new scripts and files for the mech calls computation

.gitignore

@@ -21,6 +21,7 @@ lib64/
 parts/
 sdist/
 var/
+tmp/
 wheels/
 share/python-wheels/
 *.egg-info/

app.py

@@ -136,8 +136,8 @@ demo = gr.Blocks()
 weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
     traders_data
 )
-weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
-    traders_data, trader_filter="non_Olas"
+weekly_o_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
+    traders_data, trader_filter="Olas"
 )
 weekly_non_olas_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
     traders_data, trader_filter="non_Olas"
@@ -145,16 +145,15 @@ weekly_non_olas_metrics_by_market_creator = compute_weekly_metrics_by_market_cre
 weekly_unknown_trader_metrics_by_market_creator = None
 if len(unknown_traders) > 0:
     weekly_unknown_trader_metrics_by_market_creator = (
-        compute_weekly_metrics_by_market_creator(unknown_traders)
+        compute_weekly_metrics_by_market_creator(
+            unknown_traders, trader_filter=None, unknown_trader=True
+        )
     )
 
 weekly_winning_metrics = compute_winning_metrics_by_trader(traders_data=traders_data)
 weekly_non_olas_winning_metrics = compute_winning_metrics_by_trader(
     traders_data=traders_data, trader_filter="non_Olas"
 )
-weekly_non_Olas_winning_metrics = compute_winning_metrics_by_trader(
-    traders_data=traders_data, trader_filter="non_Olas"
-)
 
 with demo:
     gr.HTML("<h1>Traders monitoring dashboard </h1>")
@@ -205,7 +204,7 @@ with demo:
                 with gr.Column(scale=3):
                     o_trader_markets_plot = plot_trader_metrics_by_market_creator(
                         metric_name=default_trader_metric,
-                        traders_df=weekly_metrics_by_market_creator,
+                        traders_df=weekly_o_metrics_by_market_creator,
                     )
                 with gr.Column(scale=1):
                     trade_details_text = get_metrics_text()
@@ -213,7 +212,7 @@ with demo:
             def update_a_trader_details(trader_detail):
                 return plot_trader_metrics_by_market_creator(
                     metric_name=trader_detail,
-                    traders_df=weekly_metrics_by_market_creator,
+                    traders_df=weekly_o_metrics_by_market_creator,
                 )
 
             trader_o_details_selector.change(
@@ -500,7 +499,7 @@ with demo:
                 metrics_text = get_metrics_text()
             with gr.Row():
                 winning_metric = plot_winning_metric_per_trader(
-                    weekly_non_Olas_winning_metrics
+                    weekly_non_olas_winning_metrics
                 )
 
 demo.queue(default_concurrency_limit=40).launch()

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4e3e1e0f256e5b3d4f25d9ecebc55ec8b80d6544e3be47aa68d36ef5a753780
-size 229899
+oid sha256:fed76273653048f900faca2d612b07f42be43d076238f0dac7f30e8882a1ec1b
+size 374565

data/unknown_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54312d1642f60df1ae5e0b13bc2ba283b0fa1216c3f5b776478e54b62b34cced
-size 11673
+oid sha256:0ab41a7a35d8bf5c588b95849ec650e048578ddcbb18bc62df0e7a3c96902ea5
+size 368142

data/weekly_mech_calls.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ae188cdae0c99e21307bddca6df7f914f376ec1d940929d7f8c2f2626aab6b
+size 59309

notebooks/num_mech_calls.ipynb

@@ -0,0 +1,326 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import gc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "weekly_mech_calls = pd.read_parquet(\"../data/weekly_mech_calls.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tools = pd.read_parquet(\"../tmp/tools.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fpmmTrades = pd.read_parquet(\"../data/fpmmTrades.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['request_id', 'request_block', 'prompt_request', 'tool', 'nonce',\n",
+       "       'trader_address', 'deliver_block', 'error', 'error_message',\n",
+       "       'prompt_response', 'mech_address', 'p_yes', 'p_no', 'confidence',\n",
+       "       'info_utility', 'vote', 'win_probability', 'market_creator', 'title',\n",
+       "       'currentAnswer', 'request_time', 'request_month_year',\n",
+       "       'request_month_year_week'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tools.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>month_year_week</th>\n",
+       "      <th>total_trades</th>\n",
+       "      <th>total_mech_calls</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0x75c0366bd0cbc3db434fd117267e32f26c5ed857</td>\n",
+       "      <td>Sep-08</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0x75c0366bd0cbc3db434fd117267e32f26c5ed857</td>\n",
+       "      <td>Sep-15</td>\n",
+       "      <td>103</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0x75c0366bd0cbc3db434fd117267e32f26c5ed857</td>\n",
+       "      <td>Sep-22</td>\n",
+       "      <td>136</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0x75c0366bd0cbc3db434fd117267e32f26c5ed857</td>\n",
+       "      <td>Sep-29</td>\n",
+       "      <td>165</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0x75c0366bd0cbc3db434fd117267e32f26c5ed857</td>\n",
+       "      <td>Oct-06</td>\n",
+       "      <td>51</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               trader_address month_year_week  total_trades  \\\n",
+       "0  0x75c0366bd0cbc3db434fd117267e32f26c5ed857          Sep-08             2   \n",
+       "1  0x75c0366bd0cbc3db434fd117267e32f26c5ed857          Sep-15           103   \n",
+       "2  0x75c0366bd0cbc3db434fd117267e32f26c5ed857          Sep-22           136   \n",
+       "3  0x75c0366bd0cbc3db434fd117267e32f26c5ed857          Sep-29           165   \n",
+       "4  0x75c0366bd0cbc3db434fd117267e32f26c5ed857          Oct-06            51   \n",
+       "\n",
+       "   total_mech_calls  \n",
+       "0                 0  \n",
+       "1                 0  \n",
+       "2                 0  \n",
+       "3                 0  \n",
+       "4                 0  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weekly_mech_calls.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>month_year_week</th>\n",
+       "      <th>total_trades</th>\n",
+       "      <th>total_mech_calls</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>14363</th>\n",
+       "      <td>0xf278dfdb02ecddc1214a151906426b9171460ec8</td>\n",
+       "      <td>Nov-24</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14364</th>\n",
+       "      <td>0xf278dfdb02ecddc1214a151906426b9171460ec8</td>\n",
+       "      <td>Dec-01</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14365</th>\n",
+       "      <td>0xf278dfdb02ecddc1214a151906426b9171460ec8</td>\n",
+       "      <td>Dec-08</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14366</th>\n",
+       "      <td>0xf278dfdb02ecddc1214a151906426b9171460ec8</td>\n",
+       "      <td>Dec-15</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14367</th>\n",
+       "      <td>0xf278dfdb02ecddc1214a151906426b9171460ec8</td>\n",
+       "      <td>Dec-22</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                   trader_address month_year_week  \\\n",
+       "14363  0xf278dfdb02ecddc1214a151906426b9171460ec8          Nov-24   \n",
+       "14364  0xf278dfdb02ecddc1214a151906426b9171460ec8          Dec-01   \n",
+       "14365  0xf278dfdb02ecddc1214a151906426b9171460ec8          Dec-08   \n",
+       "14366  0xf278dfdb02ecddc1214a151906426b9171460ec8          Dec-15   \n",
+       "14367  0xf278dfdb02ecddc1214a151906426b9171460ec8          Dec-22   \n",
+       "\n",
+       "       total_trades  total_mech_calls  \n",
+       "14363             0                 0  \n",
+       "14364             0                 0  \n",
+       "14365             0                 0  \n",
+       "14366             0                 0  \n",
+       "14367             1                 0  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weekly_mech_calls.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "count    14368.0\n",
+       "mean         0.0\n",
+       "std          0.0\n",
+       "min          0.0\n",
+       "25%          0.0\n",
+       "50%          0.0\n",
+       "75%          0.0\n",
+       "max          0.0\n",
+       "Name: total_mech_calls, dtype: float64"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "weekly_mech_calls.total_mech_calls.describe()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_dashboards",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/winning_perc.ipynb

@@ -2,25 +2,51 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
     "import pandas as pd\n",
-    "import sys\n",
-    "sys.path.append('..')\n",
-    "from scripts.metrics import compute_weekly_metrics_by_market_creator"
+    "# import sys\n",
+    "# sys.path.append('..')\n",
+    "# from scripts.metrics import compute_weekly_metrics_by_market_creator"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
     "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['trader_address', 'market_creator', 'trade_id', 'creation_timestamp',\n",
+       "       'title', 'market_status', 'collateral_amount', 'outcome_index',\n",
+       "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
+       "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
+       "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
+       "       'roi', 'staking', 'nr_mech_calls'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades.columns"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 6,

scripts/metrics.py

@@ -1,22 +1,18 @@
 import pandas as pd
 from tqdm import tqdm
+from scripts.num_mech_calls import (
+    get_daily_total_mech_calls,
+    get_weekly_total_mech_calls,
+)
 
 DEFAULT_MECH_FEE = 0.01  # xDAI
 
 
-def compute_total_nr_mech_calls_per_trader(trader_data: pd.DataFrame) -> int:
-    """Function to compute the total number of mech calls for alll markets
-    that the trader bet upon"""
-    nr_mech_calls_per_market = (
-        trader_data.groupby("title")["num_mech_calls"]
-        .max()
-        .reset_index(name="nr_mech_calls_per_market")
-    )
-    return nr_mech_calls_per_market.nr_mech_calls_per_market.sum()
-
-
 def compute_metrics(
-    trader_address: str, trader_data: pd.DataFrame, live_metrics: bool = False
+    trader_address: str,
+    trader_data: pd.DataFrame,
+    live_metrics: bool = False,
+    unknown_trader: bool = False,
 ) -> dict:
 
     if len(trader_data) == 0:
@@ -26,9 +22,14 @@ def compute_metrics(
     agg_metrics = {}
     agg_metrics["trader_address"] = trader_address
     total_bet_amounts = trader_data.collateral_amount.sum()
-    total_nr_mech_calls_all_markets = compute_total_nr_mech_calls_per_trader(
-        trader_data
-    )
+    if live_metrics:
+        # the total can be computed from daily_info.parquet
+        total_nr_mech_calls_all_markets = get_daily_total_mech_calls(trader_data)
+    elif unknown_trader:
+        # num of mech calls is always zero
+        total_nr_mech_calls_all_markets = 0
+    else:
+        total_nr_mech_calls_all_markets = get_weekly_total_mech_calls(trader_data)
     previous_total = trader_data.num_mech_calls.sum()
     agg_metrics["bet_amount"] = total_bet_amounts
     agg_metrics["nr_mech_calls"] = total_nr_mech_calls_all_markets
@@ -63,6 +64,7 @@ def compute_trader_metrics_by_market_creator(
     traders_data: pd.DataFrame,
     market_creator: str = "all",
     live_metrics: bool = False,
+    unknown_trader: bool = False,
 ) -> dict:
     """This function computes for a specific time window (week or day) the different metrics:
     roi, net_earnings, earnings, bet_amount, nr_mech_calls and nr_trades.
@@ -81,17 +83,23 @@ def compute_trader_metrics_by_market_creator(
             # tqdm.write(f"No data. Skipping market creator {market_creator}")
             return {}  # No Data
 
-    metrics = compute_metrics(trader_address, filtered_traders_data, live_metrics)
+    metrics = compute_metrics(
+        trader_address, filtered_traders_data, live_metrics, unknown_trader
+    )
     return metrics
 
 
 def merge_trader_weekly_metrics(
-    trader: str, weekly_data: pd.DataFrame, week: str
+    trader: str, weekly_data: pd.DataFrame, week: str, unknown_trader: bool = False
 ) -> pd.DataFrame:
     trader_metrics = []
     # computation as specification 1 for all types of markets
     weekly_metrics_all = compute_trader_metrics_by_market_creator(
-        trader, weekly_data, market_creator="all"
+        trader,
+        weekly_data,
+        market_creator="all",
+        live_metrics=False,
+        unknown_trader=unknown_trader,
     )
     weekly_metrics_all["month_year_week"] = week
     weekly_metrics_all["market_creator"] = "all"
@@ -99,7 +107,11 @@ def merge_trader_weekly_metrics(
 
     # computation as specification 1 for quickstart markets
     weekly_metrics_qs = compute_trader_metrics_by_market_creator(
-        trader, weekly_data, market_creator="quickstart"
+        trader,
+        weekly_data,
+        market_creator="quickstart",
+        live_metrics=False,
+        unknown_trader=unknown_trader,
     )
     if len(weekly_metrics_qs) > 0:
         weekly_metrics_qs["month_year_week"] = week
@@ -107,7 +119,11 @@ def merge_trader_weekly_metrics(
         trader_metrics.append(weekly_metrics_qs)
     # computation as specification 1 for pearl markets
     weekly_metrics_pearl = compute_trader_metrics_by_market_creator(
-        trader, weekly_data, market_creator="pearl"
+        trader,
+        weekly_data,
+        market_creator="pearl",
+        live_metrics=False,
+        unknown_trader=unknown_trader,
     )
     if len(weekly_metrics_pearl) > 0:
         weekly_metrics_pearl["month_year_week"] = week
@@ -168,7 +184,7 @@ def win_metrics_trader_level(weekly_data):
 
 
 def compute_weekly_metrics_by_market_creator(
-    traders_data: pd.DataFrame, trader_filter: str = None
+    traders_data: pd.DataFrame, trader_filter: str = None, unknown_trader: bool = False
 ) -> pd.DataFrame:
     """Function to compute the metrics at the trader level per week
     and with different categories by market creator"""
@@ -181,7 +197,11 @@ def compute_weekly_metrics_by_market_creator(
         traders = list(weekly_data.trader_address.unique())
         for trader in tqdm(traders, desc=f"Trader' metrics", unit="metrics"):
             if trader_filter is None:
-                contents.append(merge_trader_weekly_metrics(trader, weekly_data, week))
+                contents.append(
+                    merge_trader_weekly_metrics(
+                        trader, weekly_data, week, unknown_trader
+                    )
+                )
             elif trader_filter == "Olas":
                 filtered_data = weekly_data.loc[weekly_data["staking"] != "non_Olas"]
                 contents.append(

scripts/num_mech_calls.py

@@ -0,0 +1,124 @@
+import pandas as pd
+from scripts.utils import DATA_DIR, TMP_DIR
+from datetime import datetime, timezone
+from tqdm import tqdm
+
+
+def transform_to_datetime(x):
+    return datetime.fromtimestamp(int(x), tz=timezone.utc)
+
+
+def get_daily_total_mech_calls(trader_data: pd.DataFrame) -> int:
+    """Function to compute the total daily number of mech calls for all markets
+    that the trader bet upon"""
+    daily_markets = trader_data.title.unique()
+    trading_day = trader_data.creation_date.unique()
+    if len(trading_day) > 1:
+        raise ValueError("The trader data should contain only one day information")
+    total_mech_calls = 0
+    for market in daily_markets:
+        # in num_mech_calls we have the total mech calls done for that market that day
+        total_mech_calls_on_market = trader_data.loc[
+            trader_data["title"] == market, "num_mech_calls"
+        ].iloc[0]
+        total_mech_calls += total_mech_calls_on_market
+    return total_mech_calls
+
+
+def get_weekly_total_mech_calls(trader_data: pd.DataFrame) -> int:
+    """Function to compute the total weekly number of mech calls for all markets
+    that the trader bet upon"""
+    try:
+        all_mech_calls_df = pd.read_parquet(DATA_DIR / "weekly_mech_calls.parquet")
+    except Exception:
+        print("Error reading the weekly_mech_calls file")
+
+    trading_weeks = trader_data.month_year_week.unique()
+    trader_address = trader_data.trader_address.unique()[0]
+    if len(trading_weeks) > 1:
+        raise ValueError("The trader data should contain only one week information")
+    trading_week = trading_weeks[0]
+    return all_mech_calls_df.loc[
+        (all_mech_calls_df["trader_address"] == trader_address)
+        & (all_mech_calls_df["month_year_week"] == trading_week),
+        "total_mech_calls",
+    ].iloc[0]
+
+
+def compute_weekly_total_mech_calls(
+    trader: str, week: str, weekly_trades: pd.DataFrame, weekly_tools: pd.DataFrame
+) -> dict:
+    weekly_total_mech_calls_dict = {}
+    weekly_total_mech_calls_dict["trader_address"] = trader
+    weekly_total_mech_calls_dict["month_year_week"] = week
+    weekly_total_mech_calls_dict["total_trades"] = len(weekly_trades)
+    weekly_total_mech_calls_dict["total_mech_calls"] = len(weekly_tools)
+    return weekly_total_mech_calls_dict
+
+
+def compute_total_mech_calls():
+    """Function to compute the total number of mech calls for all traders and all markets
+    at a weekly level"""
+    try:
+        print("Reading tools file")
+        tools = pd.read_parquet(TMP_DIR / "tools.parquet")
+        tools["request_time"] = pd.to_datetime(tools["request_time"])
+        tools["request_date"] = tools["request_time"].dt.date
+        tools = tools.sort_values(by="request_time", ascending=True)
+        tools["month_year_week"] = (
+            tools["request_time"].dt.to_period("W").dt.strftime("%b-%d")
+        )
+
+    except Exception as e:
+        print(f"Error updating the invalid trades parquet {e}")
+
+    try:
+        print("Reading trades weekly info file")
+        fpmmTrades = pd.read_parquet(DATA_DIR / "fpmmTrades.parquet")
+        fpmmTrades["creationTimestamp"] = fpmmTrades["creationTimestamp"].apply(
+            lambda x: transform_to_datetime(x)
+        )
+        fpmmTrades["creation_timestamp"] = pd.to_datetime(
+            fpmmTrades["creationTimestamp"]
+        )
+        fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
+        fpmmTrades = fpmmTrades.sort_values(by="creation_timestamp", ascending=True)
+        fpmmTrades["month_year_week"] = (
+            fpmmTrades["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d")
+        )
+
+    except Exception as e:
+        print(f"Error reading fpmmTrades parquet {e}")
+
+    nr_traders = len(fpmmTrades["trader_address"].unique())
+    all_mech_calls = []
+    for trader in tqdm(
+        fpmmTrades["trader_address"].unique(),
+        total=nr_traders,
+        desc="creating mech calls estimation dataframe",
+    ):
+        # compute the mech calls estimations for each trader
+        all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]
+        all_tools = tools[tools["trader_address"] == trader]
+        weeks = fpmmTrades.month_year_week.unique()
+
+        for week in weeks:
+            weekly_trades = all_trades.loc[all_trades["month_year_week"] == week]
+            weekly_tools = all_tools.loc[all_tools["month_year_week"] == week]
+
+            weekly_mech_calls_dict = compute_weekly_total_mech_calls(
+                trader, week, weekly_trades, weekly_tools
+            )
+            all_mech_calls.append(weekly_mech_calls_dict)
+
+    all_mech_calls_df: pd.DataFrame = pd.DataFrame.from_dict(
+        all_mech_calls, orient="columns"
+    )
+    print("Saving weekly_mech_calls.parquet file")
+    print(all_mech_calls_df.total_mech_calls.describe())
+
+    all_mech_calls_df.to_parquet(DATA_DIR / "weekly_mech_calls.parquet", index=False)
+
+
+if __name__ == "__main__":
+    compute_total_mech_calls()

scripts/utils.py

@@ -1,4 +1,10 @@
 import pandas as pd
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).parent
+ROOT_DIR = SCRIPTS_DIR.parent
+DATA_DIR = ROOT_DIR / "data"
+TMP_DIR = ROOT_DIR / "tmp"
 
 
 def get_traders_family(row: pd.DataFrame) -> str: