RNN_charseq_and_longseq.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RNN_charseq_and_longseq.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMRp+LMP6L5GXbqWeMKsIty",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/khodid/2020_SAI_MONING2/blob/master/RNN_charseq_and_longseq.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGyhvFE0RFrd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bmvdgvn2RIJf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sample = \"electronic engineering\"  # 어느 문자열이든 가능하다.\n",
        "\n",
        "char_set = list(set(sample))\n",
        "char_dic = {c : i for i, c in enumerate(char_set)} # ? \n",
        "\n",
        "# hyper parameters\n",
        "dic_size = len(char_dic)\n",
        "hidden_size = len(char_dic)\n",
        "learning_rate = 0.1\n",
        "\n",
        "#data setting\n",
        "sample_idx = [char_dic[c] for c in sample]\n",
        "x_data = [sample_idx[:-1]]\n",
        "x_one_hot = [np.eye(dic_size)[x] for x in x_data]\n",
        "y_data = [sample_idx[1:]]\n",
        "\n",
        "#transform\n",
        "X = torch.FloatTensor(x_one_hot)\n",
        "Y = torch.LongTensor(y_data)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SEzhJ70GRn1a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9eb203b5-3163-4b48-a613-1283e459cc98"
      },
      "source": [
        "rnn = torch.nn.RNN(dic_size, hidden_size, batch_first=True)\n",
        "\n",
        "import torch.optim as optim\n",
        "\n",
        "criterion = torch.nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(rnn.parameters(), learning_rate)\n",
        "\n",
        "for i in range(100):\n",
        "    optimizer.zero_grad()\n",
        "    outputs, _status = rnn(X)\n",
        "    loss = criterion(outputs.view(-1, dic_size), Y.view(-1))\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    # 실제 모델이 어떻게 예측했는지 관찰하기 위한 코드.\n",
        "    result = outputs.data.numpy().argmax(axis = 2)\n",
        "    result_str = ''.join([char_set[c] for c in np.squeeze(result)])\n",
        "    print(i, \"loss: \", loss.item(), \"prediction str: \", result_str) "
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 loss:  2.281712770462036 prediction str:  nrnnegirrnenrgrrgngrg\n",
            "1 loss:  2.091365337371826 prediction str:  nennennenneneenennene\n",
            "2 loss:  1.9297457933425903 prediction str:  nenteinenneneenennine\n",
            "3 loss:  1.794550895690918 prediction str:  necteinentengininnini\n",
            "4 loss:  1.642216444015503 prediction str:  necteinentenginenring\n",
            "5 loss:  1.528751015663147 prediction str:  ngc renen engengnring\n",
            "6 loss:  1.4546129703521729 prediction str:  ngc iengn engengnreng\n",
            "7 loss:  1.3896015882492065 prediction str:  lgc rengn engingnreng\n",
            "8 loss:  1.3559606075286865 prediction str:  lgctrengn engengnreng\n",
            "9 loss:  1.3052265644073486 prediction str:  lectrengn engengnreng\n",
            "10 loss:  1.2671407461166382 prediction str:  lectrengntengengnreng\n",
            "11 loss:  1.2386850118637085 prediction str:  lectrengntengengereng\n",
            "12 loss:  1.2103509902954102 prediction str:  lectrengetengengereng\n",
            "13 loss:  1.1940165758132935 prediction str:  lectrengetengengereng\n",
            "14 loss:  1.168639898300171 prediction str:  leetrengetengengereng\n",
            "15 loss:  1.1564335823059082 prediction str:  leetrenentengengereng\n",
            "16 loss:  1.1469976902008057 prediction str:  leetrenentengengereng\n",
            "17 loss:  1.1404035091400146 prediction str:  leetrenentengengereng\n",
            "18 loss:  1.1281903982162476 prediction str:  leetrenentengengereng\n",
            "19 loss:  1.1176419258117676 prediction str:  leetrenintengengereng\n",
            "20 loss:  1.1135722398757935 prediction str:  leetrenintengengereng\n",
            "21 loss:  1.1356817483901978 prediction str:  leetrenentengeneereng\n",
            "22 loss:  1.0996503829956055 prediction str:  leetrenentengengereng\n",
            "23 loss:  1.1246263980865479 prediction str:  leetrenintengengereng\n",
            "24 loss:  1.1150476932525635 prediction str:  leetrenen engengereng\n",
            "25 loss:  1.1193575859069824 prediction str:  leetreren engengereng\n",
            "26 loss:  1.1167958974838257 prediction str:  leetrerentengeneereng\n",
            "27 loss:  1.1090418100357056 prediction str:  leetrenentengeneereng\n",
            "28 loss:  1.095632791519165 prediction str:  leetrenentengengereng\n",
            "29 loss:  1.092745065689087 prediction str:  leetrenentengengereng\n",
            "30 loss:  1.0776541233062744 prediction str:  leetrenentengengereng\n",
            "31 loss:  1.0872834920883179 prediction str:  lectrenentengeneereng\n",
            "32 loss:  1.070042371749878 prediction str:  lectrenintengengereng\n",
            "33 loss:  1.0724400281906128 prediction str:  lectrenintengengereng\n",
            "34 loss:  1.0632855892181396 prediction str:  lectrenintengengereng\n",
            "35 loss:  1.0610480308532715 prediction str:  lectrenintengengereng\n",
            "36 loss:  1.0539960861206055 prediction str:  leetrenintengengereng\n",
            "37 loss:  1.052889108657837 prediction str:  leetrenin engengereng\n",
            "38 loss:  1.048669695854187 prediction str:  leetrenin engengereng\n",
            "39 loss:  1.0446546077728271 prediction str:  leetrenin engengereng\n",
            "40 loss:  1.0445566177368164 prediction str:  lectrenin engengereng\n",
            "41 loss:  1.0436971187591553 prediction str:  lectrenin engengereng\n",
            "42 loss:  1.0412496328353882 prediction str:  lectrenin engengereng\n",
            "43 loss:  1.0381444692611694 prediction str:  leetrenin engengereng\n",
            "44 loss:  1.0343598127365112 prediction str:  leetrenin engengereng\n",
            "45 loss:  1.031602144241333 prediction str:  leetrenin engengereng\n",
            "46 loss:  1.03109610080719 prediction str:  leetrenin engengereng\n",
            "47 loss:  1.0274128913879395 prediction str:  leetrenin engengereng\n",
            "48 loss:  1.0242420434951782 prediction str:  leetrenin engingereng\n",
            "49 loss:  1.0202230215072632 prediction str:  leetrenin engineereng\n",
            "50 loss:  1.0172709226608276 prediction str:  leetrenin engingereng\n",
            "51 loss:  1.0409643650054932 prediction str:  leetrenin engineereng\n",
            "52 loss:  1.2258424758911133 prediction str:  lectreren engingeneng\n",
            "53 loss:  1.0540952682495117 prediction str:  leetreren engingereng\n",
            "54 loss:  1.085010290145874 prediction str:  leetreren engeneereng\n",
            "55 loss:  1.076172113418579 prediction str:  leetreren engeneereng\n",
            "56 loss:  1.0782232284545898 prediction str:  leetreren engeneereng\n",
            "57 loss:  1.0781972408294678 prediction str:  leetreren engeneereng\n",
            "58 loss:  1.076339840888977 prediction str:  leetreren engeneereng\n",
            "59 loss:  1.0754361152648926 prediction str:  leetrerentengengereng\n",
            "60 loss:  1.071568489074707 prediction str:  leetrerentengengereng\n",
            "61 loss:  1.0706491470336914 prediction str:  leetrerentengengereng\n",
            "62 loss:  1.0729405879974365 prediction str:  leetrerentengengereng\n",
            "63 loss:  1.0713355541229248 prediction str:  leetrerentengengereng\n",
            "64 loss:  1.066017985343933 prediction str:  leetrerentengengereng\n",
            "65 loss:  1.0624291896820068 prediction str:  leetrerectengengereng\n",
            "66 loss:  1.0611138343811035 prediction str:  leetrerectengengereng\n",
            "67 loss:  1.0577521324157715 prediction str:  leetrerectengengereng\n",
            "68 loss:  1.051952600479126 prediction str:  leetrenentengengereng\n",
            "69 loss:  1.046213984489441 prediction str:  leetrenintingengereng\n",
            "70 loss:  1.04152250289917 prediction str:  leetrenintingengereng\n",
            "71 loss:  1.0382765531539917 prediction str:  leetrenintingingereng\n",
            "72 loss:  1.0364423990249634 prediction str:  leetrenintingingereng\n",
            "73 loss:  1.0348464250564575 prediction str:  leetrenintingingereng\n",
            "74 loss:  1.0332838296890259 prediction str:  leetrenintingingereng\n",
            "75 loss:  1.0317388772964478 prediction str:  leetrenintingineereng\n",
            "76 loss:  1.0294033288955688 prediction str:  leetrenintingineereng\n",
            "77 loss:  1.026703119277954 prediction str:  leetrenintingineereng\n",
            "78 loss:  1.0251076221466064 prediction str:  leetrenintingineereng\n",
            "79 loss:  1.0246708393096924 prediction str:  lectrenentingineereng\n",
            "80 loss:  1.0244784355163574 prediction str:  lectrenentingineereng\n",
            "81 loss:  1.0237066745758057 prediction str:  lectrenentingineereng\n",
            "82 loss:  1.0221692323684692 prediction str:  lectrenentingineereng\n",
            "83 loss:  1.020509123802185 prediction str:  lectrenentingeneereng\n",
            "84 loss:  1.0196475982666016 prediction str:  leetrenintingeneereng\n",
            "85 loss:  1.0193791389465332 prediction str:  leetrenintingeneereng\n",
            "86 loss:  1.0184937715530396 prediction str:  leetrenintingineereng\n",
            "87 loss:  1.017193078994751 prediction str:  leetrenintingineereng\n",
            "88 loss:  1.016289472579956 prediction str:  leetrenintingineereng\n",
            "89 loss:  1.0153765678405762 prediction str:  lectrenintingineereng\n",
            "90 loss:  1.0140761137008667 prediction str:  lectrenintingineereng\n",
            "91 loss:  1.0133647918701172 prediction str:  lectrenintingineereng\n",
            "92 loss:  1.0126378536224365 prediction str:  lectrenintingineereng\n",
            "93 loss:  1.0117294788360596 prediction str:  lectrenintingineereng\n",
            "94 loss:  1.0108897686004639 prediction str:  lectrenintingineereng\n",
            "95 loss:  1.0098108053207397 prediction str:  lectrenintingineereng\n",
            "96 loss:  1.0087804794311523 prediction str:  lectrenintingineereng\n",
            "97 loss:  1.0080714225769043 prediction str:  lectrenintingineereng\n",
            "98 loss:  1.0072919130325317 prediction str:  lectrenintingineereng\n",
            "99 loss:  1.0066077709197998 prediction str:  lectrenintingineereng\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oDbWqWOCba6f",
        "colab_type": "text"
      },
      "source": [
        "# LongSeq\n",
        "긴 문장을 고정된 토막으로 잘라서 다루는 방법을 알아보겠다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WAdmUvNmRqr-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 그냥 LSTM 설명 문서에서 퍼와 봤음\n",
        "sentence = (\"In the above diagram, each line carries an entire vector, \"\n",
        "            \"from the output of one node to the inputs of others. \"\n",
        "            \"The pink circles represent pointwise operations, \"\n",
        "            \"like vector addition, while the yellow boxes are learned neural network layers. \")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CnQxeW1QcPn1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make dictionary\n",
        "char_set = list(set(sentence))\n",
        "char_dic = {c: i for i, c in enumerate(char_set)}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XyhDVCtrcRrO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# hyper parameters\n",
        "dic_size = len(char_dic)\n",
        "hidden_size = len(char_dic)\n",
        "sequence_length = 10  # Any arbitrary number\n",
        "learning_rate = 0.1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PGcXd2DEbiEi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_data = []\n",
        "y_data = []\n",
        "\n",
        "for i in range(0, len(sentence) - sequence_length):\n",
        "    x_str = sentence[i : i + sequence_length]\n",
        "    y_str = sentence[i+1 : i + sequence_length + 1]\n",
        "    \n",
        "    x_data.append([char_dic[c] for c in x_str])\n",
        "    y_data.append([char_dic[c] for c in y_str])\n",
        "    \n",
        "x_one_hot = [np.eye(dic_size)[x] for x in x_data]\n",
        "\n",
        "X = torch.FloatTensor(x_one_hot)\n",
        "Y = torch.LongTensor(y_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ce-YXFwwbuSm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(torch.nn.Module):\t\t# pytorch의 모듈을 '상속' 받는다 ...? \n",
        "    def __init__(self, input_dim, hidden_dim, layers):\n",
        "        super(Net, self).__init__()\n",
        "        self.rnn = torch.nn.RNN(input_dim, hidden_dim, num_layers=layers, batch_first=True)\n",
        "        self.fc = torch.nn.Linear(hidden_dim, hidden_dim, bias=True)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x, _status = self.rnn(x)\t# \n",
        "        x = self.fc(x)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Aza6xk2JbwEU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = Net(dic_size, hidden_size, 2) # layers에 2를 넣어 두 RNN 레이어를 지나도록 함."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxsP41E6bx1V",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3f254853-8f2f-4c3b-fb0a-f6dde6fbfb7d"
      },
      "source": [
        "criterion = torch.nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(net.parameters(), learning_rate)\n",
        "\n",
        "for i in range(100):\n",
        "    optimizer.zero_grad()\n",
        "    outputs = net(X)\n",
        "    loss = criterion(outputs.view(-1, dic_size), Y.view(-1))\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    # 모델이 예측한 것 해석하기\n",
        "    results = outputs.argmax(dim=2)\n",
        "    predict_str = \"\"\n",
        "   \n",
        "    for j, result in enumerate(results):\n",
        "        # if(i%30==0): print(i, j, ''.join([char_set[t] for t in result]), loss.item())\n",
        "        if j==0:\n",
        "            predict_str += ''.join([char_set[t] for t in result]) \n",
        "            # 처음에는 seq_length만큼 가져오고\n",
        "        else:\n",
        "            predict_str += char_set[result[-1]]\n",
        "            # 그 이후로는 업데이트되는 하나씩만 받아오기(맨 마지막 빼고 기존이랑 겹치기 때문)\n",
        "    print(i, \"||\", predict_str)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 || n the ybove ditgram, eecheline carries an entire vector  from the output of one nade to the iaputs of onhers. The pink circles bepresent pointwise operations, like vector addition, whele the yellow boxes are learned neural network layers. \n",
            "1 || niahelibore iinrrim,uoanhelinirooatpondoreoueohimoeaaoue aror toiriuooutoofatneuoiueuvonoieuioeopoiaidoihiri, tieuiini  inplorioioniuenaiouniouiririiination  uaowerieaaowetr opooni ooioiuiheunenptuooofet anaraouna.ueaeuptwioiuiore nirers, \n",
            "2 || n fi sobofe euamrau  ataheline eartimeeat treiis teaahre trac,fhedoreput on tn, iine fh mi  ireuts bn oiiesvn li daoat ahkkttdenepitdene.oi nthimeuox retinn., aiue feaapret.eipi fe,voeme ii  oeaax eroues baecfnrkiedvteuetneaetiome vires,, \n",
            "3 || n th  inove ehtrral, tnchesisesyar tp eanetrchrs fe irr, find nhe intn t pn nn  an e to mhe inests nf nnies   nhe ahae nankgtdene redenestvestiese.on uetiar,,,fine ee akr,tbento ne wh se ah  iekkonenoxe  arc tnrraedwne rnndnp wixs lare v. \n",
            "4 || n th  ynove witgrel, wpcoesens tor tp attcntchtssveTrrrs vntd,nhe ync rt pv,nttdwire yhr ie vnt ts nfean etd  The eite lhn lt vntdrere.t t r twi e yne ecinr,, fyn  fe prrslyh thrs, yhe t yie yelmorenoxe  nne tnrrftdwne lccvye wixs larelt. \n",
            "5 || n.the ynove ditgrel  tpceelens dar rtp toeoet pk vecwors tuyu nhe ypt rt pf vtt vodk toreie tnt ts nf dnhetd. The vinp lhkccs  ntcre w t p nntwis  yne dtiors  lins cecporsltd tosrs thest yhe yelmtsenovk  aykrtarrre wae cecaye we s larerc s\n",
            "6 || n.the eoone yhngrel  oacpelins corripp tocont rk lector, dutu nhe yrt ut onclfe pode coreie eap ts nf nnhe d. Theueede Thpels  ntpre   corwrntwe   yvc  niors. wibs decporslcdpnksst dhese,ree eelrolecoxk dark toamredrneucecedecwe s carerl s\n",
            "7 || neche eooxe ohngmel,etaypelils vorril  oncort rk cecwor, ceer dhe nrt ut px one nede threhe nae tw nf nnhe se The nenk cankle  btpse   toronntw seacde  nioss. line ceclor,lydenipse dhese the eellod lole donk eoarne rneurecvoecwerk.codertes\n",
            "8 || nethe aooxe ohngrel, orbaelios vap eln tneaatire teclor, deeu dhe art utipfcofe oete chrthe nae ps of orhess. The aunk cankles btprese eonopsewiseaoreranhorss line veclor,lnaeaiin  dhese the aelrhr boxe  onk coprhe reeuraceoecworkelarerles\n",
            "9 || nethesabove ohamram, oubhelios carrttn adeaatioe cealor, ceos dhe art utipf one oete thrthe napups of orhere. Theraunk cantles bapres utoponnewiseaorernthons, line veclor,lneoaiine theseuthesaellhr boxes one coprie waesralsoetworkelarerl. \n",
            "10 || nethesabove ahamram, aabhelhts carrttn aneiatioo ceatort cios dhe ant utipf one oowe thrthe naputs of onhere  Theraunk cantlessbapren utopointsise orernthons, line leatortanwotiin  these thesaellhr toxes ane coarie waesralstetwork larerri \n",
            "11 || netiesabove aiamram, aaraelite carrion aneiatioe te torf ciom,dhe ontautiof onetoome thrthe aatuts of onhors  bheraink cintles bapres ptopointsose onerntions, liwe ve torfanwooikne ohes  the aelloribofes aat liarie waeural ret ork lorerl. \n",
            "12 || nelie anove aiagram, aaraeline carritn adenntite te torf tiem,dhe ontp t of one nime tirthe iatutw of oniors. Tie oint cingles bapres ntopointsose oneratwnns, liwe ve torfanwooiwne voise the aellir boxes aad ciarie wae ral oetwore loretc. \n",
            "13 || nelie onove aiamram, eachelint carritn ad entite ve torn tiem dhe ontp t ox one oiae tirthe oat ts of oniers. The oink cinele  beprea ntppointsese onerat ons, like ve torfpddnoiwne doise tie oellir boxe  iad oiarie  ae ral oetwire larerc. \n",
            "14 || n,lii onove ciamram, eacpeline carritn an intire ve tor, ciem dhe ontp t ofeone oide cirthe nnt ts of oniers. Tie oink cirele  aepreafntppointsese one et ons, like ve torepddnoiins doise the oellir boxe  iad liarie  ae ral eetwore carerf. \n",
            "15 || n,tii obove ciagrag, eacheline carrinasin incire ve tor, vipm nhe ontp t of one oode chrthe pnp ts of onier . The phnk circle  aecreafnt poretsis  ove er ons, like ve torepndetiin, doise the oellor boxes an  oiarie  ae ral eetwork corerc. \n",
            "16 || n,whe obove cingram, eachelire carrinp intintire ve tor, vepm nhe ontp t on one dode chrthe pnp ts of onier . The phnkscirkles aecres nt poretsis  one er ons, like ve toreandetiin, woisesthe oellor boxes an  ooarie  ae rnp eetwork torerc. \n",
            "17 || n,whe obove dingrem, eachelire carnias antintire ve tor, vnow,nhe ontp tlon one dode yhrthe pnputs of onher . Thesnhnkscerkles aepres nt poretsos  dre it ons, lik  ve tor bndetiin, voile yhe oellor boxes ane learie  aeural ne work thrers. \n",
            "18 || n,the obove dingrem,seechelire caynias antent re vector, vrow,dhe ontput on one vode chrthe nnputw of onher . Thesphnk cekcles aecres nt poretwise drernt ans, lik  vector anketiin, voile yhe oellor boxes ane learip  aeural network chrerk. \n",
            "19 || n,the obove diagram, eachelire carnias nntint re vector,svrom dhe ontput of one yowe chrthe nnputw of onher . The yhnk cerkles necres ,tpporetwise auerntians, like vector anwetiin, vhile yhe oelloreboxes ane learae  aeural oetwork carerk. \n",
            "20 || n,the above ciagram, epchelire carrias an entire vectore vnom dhe ontput of one nowe chrthe aneutw of other . The yhak cerclp  becrns ,tppoirtwese afeuatians, like vector andetiin, viile the aellor boxes ane loarne  aeural oetiore larers. \n",
            "21 || n the onove ciagram, epchelirt carrio  ad entore wectore lnom dhe antput of one oowe chrthe aneutw of other . The yhok cnrclp  becres ntppoirtwese aueuatian , like vector anditions viile the aellor boxe  are loarne  aeural oetwore larers. \n",
            "22 || n the onove ciagram, eachelire carnios ar entire wector  cnom dhe ontput of one oode chrthe aneutp of other . The ahok cercles becrek ntppoirtwise aneuatian , like vector anditiin, viile the aellor boxes are lnarne  aeural oe wore larers. \n",
            "23 || n the onove ciagram, eacheline carnios ar ent re vector, crom the ontput of one oode thrthe aneutp of other . The phok cercles neprek ntipoirtwise aneuations, like vector anditiin, wiile the oellor boxes are lnarnes aeural oe iork larers. \n",
            "24 || n the onove ciagram, eacheline carnios ar ertire vector  cntm the ontput of one oode tirthe aneuts of onher , The phok cerkles nepres ntipoirtwise ouerntions, like vector andotiin, while thesoellor boxes are lnarnes aeural ne iork larers. \n",
            "25 || n the onove diagram, eacheline carnios ad ertire vector, cntm fhe output of one oode tirthe pnputs of onhers, The phnk cerkles nepres ntipoirtwise ouerrtions, like vector additiin, while the oellor boxes ane lnarne  aeural ne work larers. \n",
            "26 || n the onove diagram, eacheline carnios an ertire vector, trom the output of one oode torthe paputw of onhers, The phnk cerkles nepres ntipoirtwise ouerrtions, like vector,additiin, while the oellor boxes ane loarne  aeural ne work larers. \n",
            "27 || n the obove diagram, eacheline carnios ar ertire vector, trom the output of one oode torthe pnputw of onher . The pink cerkles nepres nt poirtwise ouerrtien , like vector,additiin, while the oellow boxes ane lnarne  neurnl network larerse \n",
            "28 || n the obove diagram, eacheline carnios ar ertire vector  trom fhe outaut of one oode torthe pnputw of onher . The pink cercles nepres nt poirtwise ouerrtien , like vector,nfditiin, while the oellow boxes ane lnarne  neural network.larers. \n",
            "29 || n the obove diagram, eacheline carnios ar ertire vector, trom fhe outaut of one eode torthe pnputw of onher . The pink circles nepres nt poirtwise ouerrtien , like vector,afditiin, while the oellow boxes ane lnarne  neural network layers. \n",
            "30 || n the obove diagram, eacheline carnio  an ertire vector, trom fhe outaut of one oode torthe paputw of onher . The pink circles nepres nt poirtwise ouerntien , like vector,afditiin, while the oellow boxes ane loarne  aeural nedwork layers. \n",
            "31 || n the onove diagram, eacheline carrio  an ert te vector, trom the output of one oode torthe paputw of onher . The phnk circles nepres nt poirtwise ouerntiens, like vector,nfdition, while the oellow boxes ant loarne  neural oedwork layers. \n",
            "32 || n the onove diagram, eacheline carrio  an ertite vector, trom the output of one oode torthe paputw of onher . The pink circles nepres nt poirtwise operntions, like vector,afdition, while the oellow boxes ant loarne  neural oetwork layers. \n",
            "33 || n the obove diagram, eacheline carnios an nntite vector  from the output of one aode torthe paputw of onher . The pink circles nepres nt poirtwise operntions, like vector afdition, while the oellow boxessane lnarne  aeural network layers. \n",
            "34 || n the above diagram, eacheline carnios an,ertire vector  from the output of one aode torthe aaputs of onhers. The aink circles nepres nt poirtwise operntions, like vector afdition, while the oellow boxessane loarno  aeural network layers. \n",
            "35 || n the obove diagram, eacheline carnios an entire vector  from the output of one aode torthe aaputs of onhers. The aink circles nepres nt poirtwise operrtions, like vector addition, while the oellow boxessare lnarno  ae ral network layers. \n",
            "36 || n the obove diagram, eacheline carnios an entire vector  from the output of one oode torthe aaputs of onhers. The aink circles nepres nt pointwise operntions, like vector addition, while the oellow boxes ane loarno  ae ral network layers. \n",
            "37 || n the obove diagram, eacheline carnios an entire vector  from the output of one oode torthe aaputs of onhers. The aink circles nepres nt pointwise operntions, like vector addition, while the oellow boxes ane liarne  neural network layers. \n",
            "38 || n the obove diagram, eacheline carnies an entire vector  from the output of one oode torthe naputs of onhers. The aink circles nepres nt poistwise operntions, like vector addition, while the oellow boxes ane liarne  neural network layers. \n",
            "39 || n the obove diagram, eacheline carnias an entire vector  from the output of one oode torthe naputs of onhers. The nink circles nepres nt pointwise operntions, like vector addition, while the yellow boxes ane liarne  neural network layers. \n",
            "40 || n the obove diagram, eacheline carnies an entire vector  from the output of one oode torthe naputs of onhers. The nink circles nepresent pointwise yperntions, like vector addition, while the yellow boxes ane loarne  neural tetwork layers. \n",
            "41 || n the obove diagram, eucheline carries an entire vector  from the output of one oode torthe naputs of onhers. The aink circles nepresent poirtwise yuerntions, like vector addition, while the oellow boxes ane laarne  neural yetwork layers. \n",
            "42 || n the obove diagram, eucheline carries an entire vector  from the output of one oode to the naputs of others. The aink circles nepresent pointwise ouerations, like vector,addition, while the yellow boxes ane learne  neural network layers. \n",
            "43 || n the obove diagram, eucheline carries an entire vector  from the output of one node to the paputs of others. The aink circles nepresent pointwise ouerations, like vector,addition, while the yellow boxes ane learne  neural network layers. \n",
            "44 || n the obove diagram, eacheline carries an entire vector  from the output of one oode to the paputs of others. The aink circles nepresent pointwise operations, like vector,addition, while the yellow boxes ane learne  neural network layers. \n",
            "45 || n the obove diagram, each line carries an entire vector  from the output of one oode to the paputs of others. The aink circles nepresent pointwise operations, like vector addition, while the yellow boxes ane learnk  neural network layers. \n",
            "46 || n the obove diagram, each line carries an entire vector  from the output of one oode torthe paputs of others. The aink circles nepresent pointwise operations, like vector addition, while the yellow boxes ane learne  neural network layers. \n",
            "47 || n the obove diagram, each line carries an entire vector  from the output of one oode torthe paputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes ane learne  neural network layers. \n",
            "48 || n the obove diagram, each line carries an entire vector  from the output of one oode torthe paputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes ane learne  neural network layers. \n",
            "49 || n the obove diagram, eacheline carries an entire vector  from the output of one oode torthe oaputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes ane learne  neural network layers. \n",
            "50 || n the obove diagram, each line carries an entire vector  from the output of one oode torthe paputs of others. The pink circles nepresent pointwise ouerations, like vector addition, while the yellow boxes ane learne  neural network layers. \n",
            "51 || n the obove diagram, each line carries an entire vector  from the output of one node torthe paputs of others. The pink circles nepresent pointwise ouerations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "52 || n the obove diagram, each line carries an entire vector  from the output of one node torthe paputs of others. The pink circles nepresent pointwise ouerations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "53 || n the obove diagram, each line carries an entire vector, from the output of one node torthe paputs of others. The pink circles nepresent pointwise ouerations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "54 || n the obove diagram, each line carries an entire vector  from the output of one node torthe paputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "55 || n the obove diagram, each line carries an entire vector  from the output of one node to the paputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "56 || n the obove diagram, each line carries an entire vector  from the output of one node to the pnputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "57 || n the obove diagram, each line carries an entire vector  from the output of one node to the pnputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learne  neural network layers. \n",
            "58 || n the obove diagram, each line carries an entire vector  from the output of one node to the pnputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "59 || n the obove diagram, each line carries an entire vector  from the output of one node to the pnputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "60 || n the obove diagram, each line carries an entire vector  from the output of one node to the anputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "61 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "62 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "63 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "64 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "65 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "66 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles nepresent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "67 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "68 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "69 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "70 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "71 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "72 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "73 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "74 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "75 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "76 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "77 || n the obove diagram, each line carries nn entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "78 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "79 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "80 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "81 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "82 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "83 || n the obove diagram, each line carries an entire vector, from the yutput of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "84 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "85 || n the obove diagram, each line carries an entire vector, lrom the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "86 || n the ybove diagram, each line carries an entire vector  from the yutput of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "87 || n the obove diagram, each line carries an entire vector  from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "88 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector,addition, while the yellow boxes are learned neural network layers. \n",
            "89 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "90 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "91 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "92 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "93 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "94 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector,addition, while the yellow boxes are learned neural network layers. \n",
            "95 || n the obove diagram, each line carries an entire vector  from the yutput of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "96 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "97 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "98 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n",
            "99 || n the obove diagram, each line carries an entire vector, from the output of one node to the inputs of others. The pink circles represent pointwise operations, like vector addition, while the yellow boxes are learned neural network layers. \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rxLqWx_xcUU_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}