updating scripts and weekly data

app.py

@@ -285,19 +285,15 @@ with demo:
                     outputs=trader_u_markets_plot,
                 )
         with gr.TabItem("📅 Daily metrics"):
-            current_week_trades = get_current_week_data(trades_df=traders_data)
             live_trades_current_week = get_current_week_data(trades_df=daily_info)
-            if len(current_week_trades) > 0:
-                daily_prof_metrics_by_market_creator = (
-                    compute_daily_metrics_by_market_creator(current_week_trades)
+            if len(live_trades_current_week) > 0:
+                live_metrics_by_market_creator = (
+                    compute_daily_metrics_by_market_creator(
+                        live_trades_current_week, trader_filter=None, live_metrics=True
+                    )
                 )
             else:
-                print("No profitability info about the current week")
-                daily_prof_metrics_by_market_creator = pd.DataFrame()
-            live_metrics_by_market_creator = compute_daily_metrics_by_market_creator(
-                live_trades_current_week, trader_filter=None, live_metrics=True
-            )
-
+                live_metrics_by_market_creator = pd.DataFrame()
             with gr.Row():
                 gr.Markdown("# Daily live metrics for all trades")
             with gr.Row():

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cfe32760dc2f816c44b5807d6fab6f079c99086a916432b5793328480491fe95
-size 69899
+oid sha256:3eced4105a55dfb65e767f6ca4dd876a9d5ff0f0247b8ca613f5cad5fe00d7e7
+size 56966

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fed76273653048f900faca2d612b07f42be43d076238f0dac7f30e8882a1ec1b
-size 374565
+oid sha256:d9f224f954dd108e164b12763dd628e05a5f17a94fd2422d9853f60f470a690d
+size 697569

data/unknown_daily_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71e9abe7b6eab8e947f6f8b9e8da398317416c7b563080023d175752062e1650
-size 30651
+oid sha256:45f38b60d39b31a089052b97cddd6fe27bf94b64f62b1eeab5947881fa590665
+size 38656

data/unknown_traders.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ab41a7a35d8bf5c588b95849ec650e048578ddcbb18bc62df0e7a3c96902ea5
-size 368142
+oid sha256:0164ef5ecaf966a5dcc677d96bba860c344f43cf53e237b6687b797502bd5e36
+size 184719

data/weekly_mech_calls.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7ae188cdae0c99e21307bddca6df7f914f376ec1d940929d7f8c2f2626aab6b
-size 59309
+oid sha256:06cafb7b09261c509c443fce63f245203341c691eeda2e4ade1d2d1b69335818
+size 55489

notebooks/unknown_traders.ipynb

@@ -0,0 +1,303 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import gc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "unknown_traders = pd.read_parquet(\"../data/unknown_traders.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>trade_id</th>\n",
+       "      <th>creation_timestamp</th>\n",
+       "      <th>title</th>\n",
+       "      <th>market_status</th>\n",
+       "      <th>collateral_amount</th>\n",
+       "      <th>outcome_index</th>\n",
+       "      <th>trade_fee_amount</th>\n",
+       "      <th>outcomes_tokens_traded</th>\n",
+       "      <th>...</th>\n",
+       "      <th>is_invalid</th>\n",
+       "      <th>winning_trade</th>\n",
+       "      <th>earnings</th>\n",
+       "      <th>redeemed</th>\n",
+       "      <th>redeemed_amount</th>\n",
+       "      <th>num_mech_calls</th>\n",
+       "      <th>mech_fee_amount</th>\n",
+       "      <th>net_earnings</th>\n",
+       "      <th>roi</th>\n",
+       "      <th>staking</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0x23522318aebb55b55879b60fb3ad4682abc6cc2f</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>0xeeaadfc4d0ef3f3bb1c430bc96657ed01a52e3e20x23...</td>\n",
+       "      <td>2024-12-23 12:36:05+00:00</td>\n",
+       "      <td>Will any additional Amazon facilities join the...</td>\n",
+       "      <td>CLOSED</td>\n",
+       "      <td>0.401540</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.004015</td>\n",
+       "      <td>0.939802</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.939802</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.939802</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.534247</td>\n",
+       "      <td>1.317323</td>\n",
+       "      <td>non_Olas</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0x8c4abc95e0091bf3bffe723d2b3c958edf642549</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>0x6df8ac2c78c8a801d6b6f30e19d3c193daf54f1e0x8c...</td>\n",
+       "      <td>2024-12-24 07:22:50+00:00</td>\n",
+       "      <td>Will the World Health Organization issue an of...</td>\n",
+       "      <td>CLOSED</td>\n",
+       "      <td>0.298503</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.002985</td>\n",
+       "      <td>0.624681</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.624681</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.624681</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.323193</td>\n",
+       "      <td>1.071994</td>\n",
+       "      <td>non_Olas</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0xb3ead49f4797662511816d2798f774dee3603185</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>0x4cb63dbf490e1f8f7c10d1e62be7ae6bbbb6d0790xb3...</td>\n",
+       "      <td>2024-12-23 12:50:40+00:00</td>\n",
+       "      <td>Will Dominion Energy announce any additional p...</td>\n",
+       "      <td>CLOSED</td>\n",
+       "      <td>0.423050</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.004230</td>\n",
+       "      <td>0.687175</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-0.427280</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>non_Olas</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0x2dd9f5678484c1f59f97ed334725858b938b4102</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>0x005e5be235ba39c5f17622d25e77557ee79a2cba0x2d...</td>\n",
+       "      <td>2024-12-22 02:40:35+00:00</td>\n",
+       "      <td>Will the Federal Reserve signal a plan to slow...</td>\n",
+       "      <td>CLOSED</td>\n",
+       "      <td>1.568561</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.015686</td>\n",
+       "      <td>2.474096</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>-1.584247</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>non_Olas</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0x2dd9f5678484c1f59f97ed334725858b938b4102</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>0x0080b3768232e8a2f187eaaf342923034275e0b90x2d...</td>\n",
+       "      <td>2024-12-13 04:32:35+00:00</td>\n",
+       "      <td>Will Russia officially confirm Bashar al-Assad...</td>\n",
+       "      <td>CLOSED</td>\n",
+       "      <td>2.677632</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.026776</td>\n",
+       "      <td>5.135035</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.135035</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.135035</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>2.430627</td>\n",
+       "      <td>0.898765</td>\n",
+       "      <td>non_Olas</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 21 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               trader_address market_creator  \\\n",
+       "0  0x23522318aebb55b55879b60fb3ad4682abc6cc2f     quickstart   \n",
+       "1  0x8c4abc95e0091bf3bffe723d2b3c958edf642549     quickstart   \n",
+       "2  0xb3ead49f4797662511816d2798f774dee3603185     quickstart   \n",
+       "3  0x2dd9f5678484c1f59f97ed334725858b938b4102     quickstart   \n",
+       "4  0x2dd9f5678484c1f59f97ed334725858b938b4102     quickstart   \n",
+       "\n",
+       "                                            trade_id  \\\n",
+       "0  0xeeaadfc4d0ef3f3bb1c430bc96657ed01a52e3e20x23...   \n",
+       "1  0x6df8ac2c78c8a801d6b6f30e19d3c193daf54f1e0x8c...   \n",
+       "2  0x4cb63dbf490e1f8f7c10d1e62be7ae6bbbb6d0790xb3...   \n",
+       "3  0x005e5be235ba39c5f17622d25e77557ee79a2cba0x2d...   \n",
+       "4  0x0080b3768232e8a2f187eaaf342923034275e0b90x2d...   \n",
+       "\n",
+       "         creation_timestamp  \\\n",
+       "0 2024-12-23 12:36:05+00:00   \n",
+       "1 2024-12-24 07:22:50+00:00   \n",
+       "2 2024-12-23 12:50:40+00:00   \n",
+       "3 2024-12-22 02:40:35+00:00   \n",
+       "4 2024-12-13 04:32:35+00:00   \n",
+       "\n",
+       "                                               title market_status  \\\n",
+       "0  Will any additional Amazon facilities join the...        CLOSED   \n",
+       "1  Will the World Health Organization issue an of...        CLOSED   \n",
+       "2  Will Dominion Energy announce any additional p...        CLOSED   \n",
+       "3  Will the Federal Reserve signal a plan to slow...        CLOSED   \n",
+       "4  Will Russia officially confirm Bashar al-Assad...        CLOSED   \n",
+       "\n",
+       "   collateral_amount outcome_index  trade_fee_amount  outcomes_tokens_traded  \\\n",
+       "0           0.401540             0          0.004015                0.939802   \n",
+       "1           0.298503             0          0.002985                0.624681   \n",
+       "2           0.423050             1          0.004230                0.687175   \n",
+       "3           1.568561             1          0.015686                2.474096   \n",
+       "4           2.677632             0          0.026776                5.135035   \n",
+       "\n",
+       "   ...  is_invalid  winning_trade  earnings  redeemed  redeemed_amount  \\\n",
+       "0  ...       False           True  0.939802      True         0.939802   \n",
+       "1  ...       False           True  0.624681      True         0.624681   \n",
+       "2  ...       False          False  0.000000     False         0.000000   \n",
+       "3  ...       False          False  0.000000      True         0.000000   \n",
+       "4  ...       False           True  5.135035      True         5.135035   \n",
+       "\n",
+       "   num_mech_calls  mech_fee_amount  net_earnings       roi   staking  \n",
+       "0               0              0.0      0.534247  1.317323  non_Olas  \n",
+       "1               0              0.0      0.323193  1.071994  non_Olas  \n",
+       "2               0              0.0     -0.427280 -1.000000  non_Olas  \n",
+       "3               0              0.0     -1.584247 -1.000000  non_Olas  \n",
+       "4               0              0.0      2.430627  0.898765  non_Olas  \n",
+       "\n",
+       "[5 rows x 21 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "unknown_traders.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1568"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(unknown_traders)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_dashboards",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scripts/closed_markets_divergence.py

@@ -85,9 +85,9 @@ def collect_liquidity_info(
     if not tokens_info:
         return None
 
-    # the second item is the final information of the market
-    first_info = tokens_info[1]
-    token_amounts = [int(x) for x in first_info["outcomeTokenAmounts"]]
+    # the last item is the final information of the market
+    last_info = tokens_info[-1]
+    token_amounts = [int(x) for x in last_info["outcomeTokenAmounts"]]
     time.sleep(IPFS_POLL_INTERVAL)
     return {fpmm_id: token_amounts}
 

scripts/metrics.py

@@ -29,7 +29,7 @@ def compute_metrics(
         total_nr_mech_calls_all_markets = 0
     else:
         total_nr_mech_calls_all_markets = get_weekly_total_mech_calls(trader_data)
-    previous_total = trader_data.num_mech_calls.sum()
+
     agg_metrics["bet_amount"] = total_bet_amounts
     agg_metrics["nr_mech_calls"] = total_nr_mech_calls_all_markets
     agg_metrics["staking"] = trader_data.iloc[0].staking
@@ -39,22 +39,16 @@ def compute_metrics(
     total_earnings = trader_data.earnings.sum()
     agg_metrics["earnings"] = total_earnings
     total_fee_amounts = trader_data.mech_fee_amount.sum()
-    previous_costs = (
-        total_bet_amounts + total_fee_amounts + previous_total * DEFAULT_MECH_FEE
-    )
+
     total_costs = (
         total_bet_amounts
         + total_fee_amounts
         + (total_nr_mech_calls_all_markets * DEFAULT_MECH_FEE)
     )
     total_net_earnings = total_earnings - total_costs
-    previous_net_earnings = trader_data.net_earnings.sum()
     agg_metrics["net_earnings"] = total_net_earnings
     agg_metrics["roi"] = total_net_earnings / total_costs
-    agg_metrics["previous_roi"] = previous_net_earnings / previous_costs
-    agg_metrics["roi_diff_perc"] = 100.0 * (
-        (agg_metrics["roi"] - agg_metrics["previous_roi"]) / abs(agg_metrics["roi"])
-    )
+
     return agg_metrics
 
 

scripts/num_mech_calls.py

@@ -1,5 +1,10 @@
 import pandas as pd
-from scripts.utils import DATA_DIR, TMP_DIR
+
+try:
+    from utils import DATA_DIR, TMP_DIR
+except ImportError:
+    from scripts.utils import DATA_DIR, TMP_DIR
+
 from datetime import datetime, timezone
 from tqdm import tqdm
 
@@ -55,30 +60,28 @@ def compute_total_mech_calls():
     except Exception as e:
         print(f"Error updating the invalid trades parquet {e}")
 
+    print("Reading trades weekly info file")
+    fpmmTrades = pd.read_parquet(DATA_DIR / "fpmmTrades.parquet")
     try:
-        print("Reading trades weekly info file")
-        fpmmTrades = pd.read_parquet(DATA_DIR / "fpmmTrades.parquet")
         fpmmTrades["creationTimestamp"] = fpmmTrades["creationTimestamp"].apply(
             lambda x: transform_to_datetime(x)
         )
-        fpmmTrades["creation_timestamp"] = pd.to_datetime(
-            fpmmTrades["creationTimestamp"]
-        )
-        fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
-        fpmmTrades = fpmmTrades.sort_values(by="creation_timestamp", ascending=True)
-        fpmmTrades["month_year_week"] = (
-            fpmmTrades["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d")
-        )
-
     except Exception as e:
-        print(f"Error reading fpmmTrades parquet {e}")
+        print(f"Transformation not needed")
+
+    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
+    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
+    fpmmTrades = fpmmTrades.sort_values(by="creation_timestamp", ascending=True)
+    fpmmTrades["month_year_week"] = (
+        fpmmTrades["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d")
+    )
 
     nr_traders = len(fpmmTrades["trader_address"].unique())
     all_mech_calls = []
     for trader in tqdm(
         fpmmTrades["trader_address"].unique(),
         total=nr_traders,
-        desc="creating mech calls estimation dataframe",
+        desc="creating weekly mech calls dataframe",
     ):
         # compute the mech calls estimations for each trader
         all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]