Nystrom.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tmukande-debug/MBHT-KDD22/blob/main/Nystrom.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ElLlO3unYYg",
        "outputId": "3fc2b787-ed4f-4a7d-9f00-b368d665a1d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'MBHT-KDD22'...\n",
            "remote: Enumerating objects: 689, done.\u001b[K\n",
            "remote: Counting objects: 100% (365/365), done.\u001b[K\n",
            "remote: Compressing objects: 100% (215/215), done.\u001b[K\n",
            "remote: Total 689 (delta 242), reused 231 (delta 145), pack-reused 324\u001b[K\n",
            "Receiving objects: 100% (689/689), 3.08 MiB | 10.87 MiB/s, done.\n",
            "Resolving deltas: 100% (269/269), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/tmukande-debug/MBHT-KDD22"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PvS4qS5bnwy5",
        "outputId": "4c9ffc45-dab5-4c9a-d9e3-507030f9ad15"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/MBHT-KDD22\n"
          ]
        }
      ],
      "source": [
        "%cd MBHT-KDD22"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gfq6jLOUA86_",
        "outputId": "3815049d-28e5-4a61-bc44-d4daaa4f1636"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting nystrom-attention\n",
            "  Downloading nystrom_attention-0.0.11-py3-none-any.whl (4.5 kB)\n",
            "Collecting einops>=0.3\n",
            "  Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 KB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.6 in /usr/local/lib/python3.8/dist-packages (from nystrom-attention) (1.13.1+cu116)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch>=1.6->nystrom-attention) (4.5.0)\n",
            "Installing collected packages: einops, nystrom-attention\n",
            "Successfully installed einops-0.6.0 nystrom-attention-0.0.11\n"
          ]
        }
      ],
      "source": [
        "! pip install nystrom-attention"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CgzZDNC_oP-G",
        "outputId": "0d95bff3-8782-40ab-9652-0b2e71da6a46"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch>=1.7.0 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 1)) (1.13.1+cu116)\n",
            "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 2)) (1.22.4)\n",
            "Collecting scipy==1.6.0\n",
            "  Downloading scipy-1.6.0-cp38-cp38-manylinux1_x86_64.whl (27.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.2/27.2 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hyperopt>=0.2.4\n",
            "  Downloading hyperopt-0.2.7-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m56.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas>=1.0.5 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 5)) (1.3.5)\n",
            "Requirement already satisfied: tqdm>=4.48.2 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 6)) (4.64.1)\n",
            "Requirement already satisfied: scikit_learn>=0.23.2 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 7)) (1.0.2)\n",
            "Requirement already satisfied: pyyaml>=5.1.0 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 8)) (6.0)\n",
            "Collecting colorlog==4.7.2\n",
            "  Downloading colorlog-4.7.2-py2.py3-none-any.whl (10 kB)\n",
            "Collecting colorama==0.4.4\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tensorboard>=2.5.0 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 11)) (2.11.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch>=1.7.0->-r requirements.txt (line 1)) (4.5.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from hyperopt>=0.2.4->-r requirements.txt (line 4)) (1.15.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.8/dist-packages (from hyperopt>=0.2.4->-r requirements.txt (line 4)) (0.16.0)\n",
            "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.8/dist-packages (from hyperopt>=0.2.4->-r requirements.txt (line 4)) (3.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.8/dist-packages (from hyperopt>=0.2.4->-r requirements.txt (line 4)) (2.2.1)\n",
            "Collecting py4j\n",
            "  Downloading py4j-0.10.9.7-py2.py3-none-any.whl (200 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.5/200.5 KB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas>=1.0.5->-r requirements.txt (line 5)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas>=1.0.5->-r requirements.txt (line 5)) (2022.7.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit_learn>=0.23.2->-r requirements.txt (line 7)) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit_learn>=0.23.2->-r requirements.txt (line 7)) (3.1.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.8.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (0.4.6)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (57.4.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.51.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.4.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (3.4.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (2.16.1)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (0.6.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (0.38.4)\n",
            "Requirement already satisfied: protobuf<4,>=3.9.2 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (3.19.6)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (2.25.1)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.0.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->-r requirements.txt (line 11)) (5.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->-r requirements.txt (line 11)) (4.9)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->-r requirements.txt (line 11)) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard>=2.5.0->-r requirements.txt (line 11)) (6.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->-r requirements.txt (line 11)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->-r requirements.txt (line 11)) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->-r requirements.txt (line 11)) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->-r requirements.txt (line 11)) (2.10)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.5.0->-r requirements.txt (line 11)) (3.14.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.5.0->-r requirements.txt (line 11)) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.5.0->-r requirements.txt (line 11)) (3.2.2)\n",
            "Installing collected packages: py4j, colorlog, scipy, colorama, hyperopt\n",
            "  Attempting uninstall: scipy\n",
            "    Found existing installation: scipy 1.7.3\n",
            "    Uninstalling scipy-1.7.3:\n",
            "      Successfully uninstalled scipy-1.7.3\n",
            "  Attempting uninstall: hyperopt\n",
            "    Found existing installation: hyperopt 0.1.2\n",
            "    Uninstalling hyperopt-0.1.2:\n",
            "      Successfully uninstalled hyperopt-0.1.2\n",
            "Successfully installed colorama-0.4.4 colorlog-4.7.2 hyperopt-0.2.7 py4j-0.10.9.7 scipy-1.6.0\n"
          ]
        }
      ],
      "source": [
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QCObbjX31XZ7",
        "outputId": "c1c420c4-1b51-42ec-ee39-976765aace81"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zrSKvTf0pAm4",
        "outputId": "a7867fec-efba-4b83-a2d5-034d0cbe1ab3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "27 Feb 11:49    INFO  PID: 796\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  Namespace(batch_size=2048, dataset='retail_beh', gpu_id=0, model='[MBHT]', valid_portion=0.1, validation=False)\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  \n",
            "\u001b[1;35mGeneral Hyper Parameters:\n",
            "\u001b[0m\u001b[1;36mgpu_id\u001b[0m =\u001b[1;33m 0\u001b[0m\n",
            "\u001b[1;36muse_gpu\u001b[0m =\u001b[1;33m True\u001b[0m\n",
            "\u001b[1;36mseed\u001b[0m =\u001b[1;33m 2020\u001b[0m\n",
            "\u001b[1;36mstate\u001b[0m =\u001b[1;33m INFO\u001b[0m\n",
            "\u001b[1;36mreproducibility\u001b[0m =\u001b[1;33m True\u001b[0m\n",
            "\u001b[1;36mdata_path\u001b[0m =\u001b[1;33m /content/drive/MyDrive/RESEARCH_PROJECT/datasets/MBHT_dataset/retail_beh/\u001b[0m\n",
            "\u001b[1;36mshow_progress\u001b[0m =\u001b[1;33m True\u001b[0m\n",
            "\u001b[1;36msave_dataset\u001b[0m =\u001b[1;33m False\u001b[0m\n",
            "\u001b[1;36msave_dataloaders\u001b[0m =\u001b[1;33m False\u001b[0m\n",
            "\u001b[1;36mbenchmark_filename\u001b[0m =\u001b[1;33m ['train', 'test']\u001b[0m\n",
            "\n",
            "\u001b[1;35mTraining Hyper Parameters:\n",
            "\u001b[0m\u001b[1;36mcheckpoint_dir\u001b[0m =\u001b[1;33m saved\u001b[0m\n",
            "\u001b[1;36mepochs\u001b[0m =\u001b[1;33m 300\u001b[0m\n",
            "\u001b[1;36mtrain_batch_size\u001b[0m =\u001b[1;33m 64\u001b[0m\n",
            "\u001b[1;36mlearner\u001b[0m =\u001b[1;33m adam\u001b[0m\n",
            "\u001b[1;36mlearning_rate\u001b[0m =\u001b[1;33m 0.001\u001b[0m\n",
            "\u001b[1;36meval_step\u001b[0m =\u001b[1;33m 1\u001b[0m\n",
            "\u001b[1;36mstopping_step\u001b[0m =\u001b[1;33m 10\u001b[0m\n",
            "\u001b[1;36mclip_grad_norm\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mweight_decay\u001b[0m =\u001b[1;33m 0.0\u001b[0m\n",
            "\u001b[1;36mloss_decimal_place\u001b[0m =\u001b[1;33m 4\u001b[0m\n",
            "\n",
            "\u001b[1;35mEvaluation Hyper Parameters:\n",
            "\u001b[0m\u001b[1;36meval_args\u001b[0m =\u001b[1;33m {'mode': 'full', 'order': 'TO', 'split': {'RS': [0.8, 0.1, 0.1]}, 'group_by': 'user'}\u001b[0m\n",
            "\u001b[1;36mmetrics\u001b[0m =\u001b[1;33m ['Recall', 'NDCG', 'MRR']\u001b[0m\n",
            "\u001b[1;36mtopk\u001b[0m =\u001b[1;33m [5, 10, 101]\u001b[0m\n",
            "\u001b[1;36mvalid_metric\u001b[0m =\u001b[1;33m NDCG@10\u001b[0m\n",
            "\u001b[1;36mvalid_metric_bigger\u001b[0m =\u001b[1;33m True\u001b[0m\n",
            "\u001b[1;36meval_batch_size\u001b[0m =\u001b[1;33m 128\u001b[0m\n",
            "\u001b[1;36mmetric_decimal_place\u001b[0m =\u001b[1;33m 4\u001b[0m\n",
            "\n",
            "\u001b[1;35mDataset Hyper Parameters:\n",
            "\u001b[0m\u001b[1;36mfield_separator\u001b[0m =\u001b[1;33m \t\u001b[0m\n",
            "\u001b[1;36mseq_separator\u001b[0m =\u001b[1;33m  \u001b[0m\n",
            "\u001b[1;36mUSER_ID_FIELD\u001b[0m =\u001b[1;33m session_id\u001b[0m\n",
            "\u001b[1;36mITEM_ID_FIELD\u001b[0m =\u001b[1;33m item_id\u001b[0m\n",
            "\u001b[1;36mRATING_FIELD\u001b[0m =\u001b[1;33m rating\u001b[0m\n",
            "\u001b[1;36mTIME_FIELD\u001b[0m =\u001b[1;33m timestamp\u001b[0m\n",
            "\u001b[1;36mseq_len\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mLABEL_FIELD\u001b[0m =\u001b[1;33m label\u001b[0m\n",
            "\u001b[1;36mthreshold\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mNEG_PREFIX\u001b[0m =\u001b[1;33m neg_\u001b[0m\n",
            "\u001b[1;36mload_col\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36munload_col\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36munused_col\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36madditional_feat_suffix\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mrm_dup_inter\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mval_interval\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mfilter_inter_by_user_or_item\u001b[0m =\u001b[1;33m True\u001b[0m\n",
            "\u001b[1;36muser_inter_num_interval\u001b[0m =\u001b[1;33m [0,inf)\u001b[0m\n",
            "\u001b[1;36mitem_inter_num_interval\u001b[0m =\u001b[1;33m [0,inf)\u001b[0m\n",
            "\u001b[1;36malias_of_user_id\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36malias_of_item_id\u001b[0m =\u001b[1;33m ['item_id_list']\u001b[0m\n",
            "\u001b[1;36malias_of_entity_id\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36malias_of_relation_id\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mpreload_weight\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mnormalize_field\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mnormalize_all\u001b[0m =\u001b[1;33m None\u001b[0m\n",
            "\u001b[1;36mITEM_LIST_LENGTH_FIELD\u001b[0m =\u001b[1;33m item_length\u001b[0m\n",
            "\u001b[1;36mLIST_SUFFIX\u001b[0m =\u001b[1;33m _list\u001b[0m\n",
            "\u001b[1;36mMAX_ITEM_LIST_LENGTH\u001b[0m =\u001b[1;33m 200\u001b[0m\n",
            "\u001b[1;36mPOSITION_FIELD\u001b[0m =\u001b[1;33m position_id\u001b[0m\n",
            "\u001b[1;36mHEAD_ENTITY_ID_FIELD\u001b[0m =\u001b[1;33m head_id\u001b[0m\n",
            "\u001b[1;36mTAIL_ENTITY_ID_FIELD\u001b[0m =\u001b[1;33m tail_id\u001b[0m\n",
            "\u001b[1;36mRELATION_ID_FIELD\u001b[0m =\u001b[1;33m relation_id\u001b[0m\n",
            "\u001b[1;36mENTITY_ID_FIELD\u001b[0m =\u001b[1;33m entity_id\u001b[0m\n",
            "\n",
            "\u001b[1;35mOther Hyper Parameters: \n",
            "\u001b[0m\u001b[1;36mneg_sampling\u001b[0m = \u001b[1;33mNone\u001b[0m\n",
            "\u001b[1;36mrepeatable\u001b[0m = \u001b[1;33mTrue\u001b[0m\n",
            "\u001b[1;36mn_layers\u001b[0m = \u001b[1;33m2\u001b[0m\n",
            "\u001b[1;36mn_heads\u001b[0m = \u001b[1;33m2\u001b[0m\n",
            "\u001b[1;36mhidden_size\u001b[0m = \u001b[1;33m64\u001b[0m\n",
            "\u001b[1;36minner_size\u001b[0m = \u001b[1;33m256\u001b[0m\n",
            "\u001b[1;36mhidden_dropout_prob\u001b[0m = \u001b[1;33m0.5\u001b[0m\n",
            "\u001b[1;36mattn_dropout_prob\u001b[0m = \u001b[1;33m0.5\u001b[0m\n",
            "\u001b[1;36mhidden_act\u001b[0m = \u001b[1;33mgelu\u001b[0m\n",
            "\u001b[1;36mlayer_norm_eps\u001b[0m = \u001b[1;33m1e-12\u001b[0m\n",
            "\u001b[1;36minitializer_range\u001b[0m = \u001b[1;33m0.02\u001b[0m\n",
            "\u001b[1;36mmask_ratio\u001b[0m = \u001b[1;33m0.2\u001b[0m\n",
            "\u001b[1;36mloss_type\u001b[0m = \u001b[1;33mCE\u001b[0m\n",
            "\u001b[1;36mcustomized_eval\u001b[0m = \u001b[1;33m1\u001b[0m\n",
            "\u001b[1;36mMODEL_TYPE\u001b[0m = \u001b[1;33mModelType.SEQUENTIAL\u001b[0m\n",
            "\u001b[1;36mhyper_len\u001b[0m = \u001b[1;33m6\u001b[0m\n",
            "\u001b[1;36mscales\u001b[0m = \u001b[1;33m[5, 4, 20]\u001b[0m\n",
            "\u001b[1;36menable_hg\u001b[0m = \u001b[1;33m1\u001b[0m\n",
            "\u001b[1;36menable_ms\u001b[0m = \u001b[1;33m1\u001b[0m\n",
            "\u001b[1;36mabaltion\u001b[0m = \u001b[1;33m\u001b[0m\n",
            "\u001b[1;36mMODEL_INPUT_TYPE\u001b[0m = \u001b[1;33mInputType.POINTWISE\u001b[0m\n",
            "\u001b[1;36meval_type\u001b[0m = \u001b[1;33mEvaluatorType.RANKING\u001b[0m\n",
            "\u001b[1;36mdevice\u001b[0m = \u001b[1;33mcpu\u001b[0m\n",
            "\u001b[1;36mtrain_neg_sample_args\u001b[0m = \u001b[1;33m{'strategy': 'none'}\u001b[0m\n",
            "\u001b[1;36meval_neg_sample_args\u001b[0m = \u001b[1;33m{'strategy': 'full', 'distribution': 'uniform'}\u001b[0m\n",
            "\n",
            "\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  \u001b[1;35mretail_beh\u001b[0m\n",
            "\u001b[1;34mThe number of users\u001b[0m: 30691\n",
            "\u001b[1;34mAverage actions of users\u001b[0m: 1.065167807103291\n",
            "\u001b[1;34mThe number of items\u001b[0m: 31240\n",
            "\u001b[1;34mAverage actions of items\u001b[0m: 3.388618223281849\n",
            "\u001b[1;34mThe number of inters\u001b[0m: 32690\n",
            "\u001b[1;34mThe sparsity of the dataset\u001b[0m: 99.99659048303167%\n",
            "\u001b[1;34mRemain Fields\u001b[0m: ['session_id', 'item_id_list', 'item_type_list', 'item_id', 'item_length']\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  MBHT(\n",
            "  (type_embedding): Embedding(6, 64, padding_idx=0)\n",
            "  (item_embedding): Embedding(31241, 64, padding_idx=0)\n",
            "  (position_embedding): Embedding(201, 64)\n",
            "  (trm_encoder): TransformerEncoder(\n",
            "    (layer): ModuleList(\n",
            "      (0): TransformerLayer(\n",
            "        (multi_head_attention): MultiScaleAttention(\n",
            "          (out_fc): Linear(in_features=260, out_features=200, bias=True)\n",
            "          (attention1): LinearAttention(\n",
            "            (E): Linear(in_features=200, out_features=5, bias=True)\n",
            "            (F): Linear(in_features=200, out_features=5, bias=True)\n",
            "            (W_V): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (W_K): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (W_Q): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (dense): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (attn_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (out_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (attention2): MultiHeadAttention(\n",
            "            (query): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (key): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (value): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (attn_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (dense): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "            (out_dropout): Dropout(p=0.5, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (feed_forward): FeedForward(\n",
            "          (dense_1): Linear(in_features=64, out_features=256, bias=True)\n",
            "          (dense_2): Linear(in_features=256, out_features=64, bias=True)\n",
            "          (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.5, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (1): TransformerLayer(\n",
            "        (multi_head_attention): MultiScaleAttention(\n",
            "          (out_fc): Linear(in_features=260, out_features=200, bias=True)\n",
            "          (attention1): LinearAttention(\n",
            "            (E): Linear(in_features=200, out_features=5, bias=True)\n",
            "            (F): Linear(in_features=200, out_features=5, bias=True)\n",
            "            (W_V): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (W_K): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (W_Q): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (dense): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (attn_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (out_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (attention2): MultiHeadAttention(\n",
            "            (query): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (key): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (value): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (attn_dropout): Dropout(p=0.5, inplace=False)\n",
            "            (dense): Linear(in_features=64, out_features=64, bias=True)\n",
            "            (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "            (out_dropout): Dropout(p=0.5, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (feed_forward): FeedForward(\n",
            "          (dense_1): Linear(in_features=64, out_features=256, bias=True)\n",
            "          (dense_2): Linear(in_features=256, out_features=64, bias=True)\n",
            "          (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.5, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (hgnn_layer): HGNN(\n",
            "    (hgc1): HGNN_conv()\n",
            "    (hgc2): HGNN_conv()\n",
            "    (out_fc): Linear(in_features=64, out_features=64, bias=True)\n",
            "  )\n",
            "  (LayerNorm): LayerNorm((64,), eps=1e-12, elementwise_affine=True)\n",
            "  (dropout): Dropout(p=0.5, inplace=False)\n",
            "  (hg_type_embedding): Embedding(6, 64, padding_idx=0)\n",
            ")\u001b[1;34m\n",
            "Trainable parameters\u001b[0m: 2276036\u001b[0m\n",
            "\u001b[0m2023-02-27 11:49:37.605626: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2023-02-27 11:49:39.135161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
            "2023-02-27 11:49:39.135331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
            "2023-02-27 11:49:39.135356: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "\u001b[1;35mEvaluate   \u001b[0m: 100%|██████████████████████████████████████████████████| 16/16 [00:09<00:00,  1.61it/s]\u001b[0m\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  \u001b[1;32mepoch -1 evaluating\u001b[0m [\u001b[1;34mtime\u001b[0m: 9.93s, \u001b[1;34mvalid_score\u001b[0m: 0.054900]\u001b[0m\n",
            "\u001b[0m27 Feb 11:49    INFO  \u001b[1;34mvalid result\u001b[0m: \n",
            "recall@5 : 0.0625    recall@10 : 0.1245    recall@101 : 1.0    ndcg@5 : 0.0356    ndcg@10 : 0.0549    ndcg@101 : 0.2208    mrr@5 : 0.0271    mrr@10 : 0.0346    mrr@101 : 0.0609    \u001b[0m\n",
            "\u001b[1;35mTrain     0\u001b[0m: 100%|████████████████████████████████████████████████| 480/480 [22:56<00:00,  2.87s/it]\u001b[0m\u001b[0m\n",
            "\u001b[0m27 Feb 12:12    INFO  \u001b[1;32mepoch 0 training\u001b[0m [\u001b[1;34mtime\u001b[0m: 1376.28s, \u001b[1;34mtrain loss\u001b[0m: 4547.7902]\u001b[0m\n",
            "\u001b[1;35mEvaluate   \u001b[0m: 100%|██████████████████████████████████████████████████| 16/16 [00:09<00:00,  1.67it/s]\u001b[0m\u001b[0m\n",
            "\u001b[0m27 Feb 12:12    INFO  \u001b[1;32mepoch 0 evaluating\u001b[0m [\u001b[1;34mtime\u001b[0m: 9.60s, \u001b[1;34mvalid_score\u001b[0m: 0.667300]\u001b[0m\n",
            "\u001b[0m27 Feb 12:12    INFO  \u001b[1;34mvalid result\u001b[0m: \n",
            "recall@5 : 0.7475    recall@10 : 0.8205    recall@101 : 1.0    ndcg@5 : 0.6436    ndcg@10 : 0.6673    ndcg@101 : 0.7041    mrr@5 : 0.6089    mrr@10 : 0.6186    mrr@101 : 0.6257    \u001b[0m\n",
            "\u001b[0m27 Feb 12:12    INFO  \u001b[1;34mSaving current best\u001b[0m: saved/MBHT-retail_beh-Feb-27-2023_11-49-40.pth\u001b[0m\n",
            "\u001b[1;35mTrain     1\u001b[0m: 100%|████████████████████████████████████████████████| 480/480 [23:00<00:00,  2.88s/it]\u001b[0m\u001b[0m\n",
            "\u001b[0m27 Feb 12:35    INFO  \u001b[1;32mepoch 1 training\u001b[0m [\u001b[1;34mtime\u001b[0m: 1380.96s, \u001b[1;34mtrain loss\u001b[0m: 3721.4123]\u001b[0m\n",
            "\u001b[1;35mEvaluate   \u001b[0m: 100%|██████████████████████████████████████████████████| 16/16 [00:08<00:00,  1.80it/s]\u001b[0m\u001b[0m\n",
            "\u001b[0m27 Feb 12:36    INFO  \u001b[1;32mepoch 1 evaluating\u001b[0m [\u001b[1;34mtime\u001b[0m: 8.92s, \u001b[1;34mvalid_score\u001b[0m: 0.769100]\u001b[0m\n",
            "\u001b[0m27 Feb 12:36    INFO  \u001b[1;34mvalid result\u001b[0m: \n",
            "recall@5 : 0.804    recall@10 : 0.8415    recall@101 : 1.0    ndcg@5 : 0.7569    ndcg@10 : 0.7691    ndcg@101 : 0.7968    mrr@5 : 0.741    mrr@10 : 0.7461    mrr@101 : 0.7497    \u001b[0m\n",
            "\u001b[0m27 Feb 12:36    INFO  \u001b[1;34mSaving current best\u001b[0m: saved/MBHT-retail_beh-Feb-27-2023_11-49-40.pth\u001b[0m\n",
            "\u001b[1;35mTrain     2\u001b[0m:  47%|██████████████████████▋                         | 227/480 [10:41<12:50,  3.04s/it]\u001b[0m\u001b[0m"
          ]
        }
      ],
      "source": [
        "!python runNystrom.py --model=[MBHT] --dataset=retail_beh"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNWOqL1fne0/Di9PlBGGIRR",
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}