Merge pull request #301 from vertica/qprof_ui_main

QueryProfile Main Page - Check for Table retention when using Transaction and Statement IDs

project/ui/qprof_main.ipynb

@@ -65,6 +65,7 @@
     "logging.info('[Query Profile Main Page] Importing Libraries')\n",
     "from ipywidgets import interact, interactive, fixed, interact_manual\n",
     "import ipywidgets as widgets\n",
+    "from datetime import datetime\n",
     "import verticapy as vp\n",
     "from IPython.display import display,clear_output, IFrame, HTML\n",
     "import pickle\n",
@@ -611,6 +612,66 @@
     "target_schema_2_combo = widgets.HBox([target_schema__2, create_tooltip(\"Enter the schema in which you would like to store the profile data. Note that you will need the combination of schema and the key to load your saved data. If you do not provide this, you may not be able to export your profile tables to a tar file.\")])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd710f93-285e-4662-a722-c79cbd9559ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def check_rentention(transactions):\n",
+    "    # Initialize an empty condition string\n",
+    "    condition=\"\"\n",
+    "    # Loop through pairs and build the condition\n",
+    "    for i in range(len(transactions)):\n",
+    "      # Append each pair to the condition with an OR if not the first pair\n",
+    "      if i !=0:\n",
+    "        condition+=\" OR \"\n",
+    "      # Add the pair condition\n",
+    "      condition+=f\"(transaction_id = {transactions[i][0]} AND statement_id = {transactions[i][1]})\"\n",
+    "    # Formulate the SQL query with the constructed condition\n",
+    "    query = f\"SELECT MIN(start_timestamp) FROM query_requests WHERE {condition};\"\n",
+    "    res= vp._utils._sql._sys._executeSQL(query, method = \"fetchall\")\n",
+    "    query_min_time = res[0][0]\n",
+    "    if isinstance(res[0][0], type(None)):\n",
+    "        raise ValueError(f\"The queries with transaction id/statement ids '{transactions}' not found int the QUERY_REQUESTS table.\")\n",
+    "\n",
+    "    important_table_list = [\n",
+    "        # ['execution_engine_profiles', None], # This is crucial to have data. But do we have warning if there is no data?\n",
+    "        ['v_internal.dc_explain_plans', 'time'],\n",
+    "        ['v_internal.dc_query_executions', 'time'],\n",
+    "        ['v_internal.dc_requests_issued', 'time'],\n",
+    "        ['v_internal.dc_plan_steps', 'time'],\n",
+    "        ['v_monitor.query_events', 'event_timestamp'],\n",
+    "        # ['host_resources', None],\n",
+    "        ['v_monitor.query_consumption', 'start_time'],\n",
+    "        # ['v_monitor.query_plan_profiles', None],\n",
+    "        ['query_profiles', 'query_start'],\n",
+    "        # ['resource_pool_status', None],\n",
+    "        ['v_monitor.resource_acquisitions' , 'queue_entry_timestamp'],\n",
+    "        ['v_internal.dc_plan_activities', 'start_time']\n",
+    "    ]\n",
+    "    for table, table_time in important_table_list:\n",
+    "        res= vp._utils._sql._sys._executeSQL(f\"SELECT MIN({table_time}) from {table};\", method = \"fetchall\")\n",
+    "        table_min_time = res[0][0]\n",
+    "        if isinstance(table_min_time, str):\n",
+    "            date_string = res[0][0]\n",
+    "            # Replace '+00' with '+0000' to match the expected format\n",
+    "            try:\n",
+    "                date_string = date_string.replace('+00', '+0000')\n",
+    "                table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
+    "            except:\n",
+    "                if '+' in date_string or '-' in date_string:\n",
+    "                    # Locate the position of '+' or '-' for timezone\n",
+    "                    split_pos = max(date_string.rfind('+'), date_string.rfind('-'))\n",
+    "                    if len(date_string) - split_pos == 3:  # Handle offsets like '-05'\n",
+    "                        date_string = date_string[:split_pos] + date_string[split_pos:] + ':00'\n",
+    "                table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
+    "        print(f\"The table '{table}' has entries going far back as time {res[0][0]}\")\n",
+    "        if query_min_time<table_min_time:\n",
+    "            raise ValueError(\"Retention issues. The table '{table}' does not seem to have old enough data to capture the query profile. Please re-profile your query\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -636,6 +697,7 @@
     "            key_val_2 = key_2.value\n",
     "            transaction_val = transaction.value\n",
     "            statement_val = statement.value\n",
+    "            check_rentention(transactions = [(transaction_val,statement_val),])\n",
     "            logging.info(f\"[Query Profile Main Page] For create option, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, transaction id as {transaction_val}, statement id as {statement_val}.\")\n",
     "            reset_values(statement_val = statement_val, transaction_val = transaction_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n",
     "\n",
@@ -752,6 +814,7 @@
     "                key_val_2 = key_2.value\n",
     "                transaction_statement_list_val = transaction_statement_list.value\n",
     "                transaction_statement_list_val.replace(\" \", \"\")\n",
+    "                check_rentention(transactions = eval(transaction_statement_list_val))\n",
     "                logging.info(f\"[Query Profile Main Page] For create using multiple transaction ids and keys, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, List of transactions and statements as {transaction_statement_list_val}.\")\n",
     "                reset_values(transaction_statement_list_val = transaction_statement_list_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n",
     "\n",