Spaces:
Running
Running
Add disabled example showing how to read and write to Azure Blob Storage
Browse files- polars/03_loading_data.py +31 -3
polars/03_loading_data.py
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
|
| 14 |
import marimo
|
| 15 |
|
| 16 |
-
__generated_with = "0.13.
|
| 17 |
app = marimo.App(width="medium")
|
| 18 |
|
| 19 |
|
|
@@ -257,7 +257,6 @@ def _(mo):
|
|
| 257 |
def _(mo):
|
| 258 |
mo.md(
|
| 259 |
r"""
|
| 260 |
-
|
| 261 |
## Hive Partitions
|
| 262 |
|
| 263 |
There is also support for [Hive](https://docs.pola.rs/user-guide/io/hive/) partitioned data, but parts of the API are still unstable (may change in future polars versions
|
|
@@ -339,18 +338,47 @@ def _(mo):
|
|
| 339 |
|
| 340 |
The API is the same for all three storage providers, check the [User Guide](https://docs.pola.rs/user-guide/io/cloud-storage/) if you need of any of them.
|
| 341 |
|
| 342 |
-
|
| 343 |
"""
|
| 344 |
)
|
| 345 |
return
|
| 346 |
|
| 347 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 348 |
@app.cell
|
| 349 |
def _():
|
| 350 |
import marimo as mo
|
| 351 |
return (mo,)
|
| 352 |
|
| 353 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
@app.cell
|
| 355 |
def _():
|
| 356 |
import pathlib
|
|
|
|
| 13 |
|
| 14 |
import marimo
|
| 15 |
|
| 16 |
+
__generated_with = "0.13.15"
|
| 17 |
app = marimo.App(width="medium")
|
| 18 |
|
| 19 |
|
|
|
|
| 257 |
def _(mo):
|
| 258 |
mo.md(
|
| 259 |
r"""
|
|
|
|
| 260 |
## Hive Partitions
|
| 261 |
|
| 262 |
There is also support for [Hive](https://docs.pola.rs/user-guide/io/hive/) partitioned data, but parts of the API are still unstable (may change in future polars versions
|
|
|
|
| 338 |
|
| 339 |
The API is the same for all three storage providers, check the [User Guide](https://docs.pola.rs/user-guide/io/cloud-storage/) if you need of any of them.
|
| 340 |
|
| 341 |
+
Runnable examples are not included in this Notebook as it would require setting up authentication, but the disabled cell below shows an example using Azure.
|
| 342 |
"""
|
| 343 |
)
|
| 344 |
return
|
| 345 |
|
| 346 |
|
| 347 |
+
@app.cell(disabled=True)
|
| 348 |
+
def _(adlfs, df, os, pl):
|
| 349 |
+
fs = adlfs.AzureBlobFileSystem(connection_string=os.environ["AZURE_STORAGE_CONNECTION_STRING"])
|
| 350 |
+
destination = f"abfs://{os.environ['AZURE_CONTAINER_NAME']}/file.parquet"
|
| 351 |
+
|
| 352 |
+
# Writing
|
| 353 |
+
with fs.open(destination, mode="wb") as f:
|
| 354 |
+
df.write_parquet(f)
|
| 355 |
+
|
| 356 |
+
# Reading
|
| 357 |
+
pl.read_parquet(
|
| 358 |
+
destination, storage_options={"account_name": os.environ["AZURE_STORAGE_ACCOUNT"], "use_azure_cli": "True"}
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
# Deleting
|
| 362 |
+
fs.delete(destination)
|
| 363 |
+
|
| 364 |
+
# If you get an error saying that the account does not exists, double check you logged in the correct account and subscription via `az login`
|
| 365 |
+
return
|
| 366 |
+
|
| 367 |
+
|
| 368 |
@app.cell
|
| 369 |
def _():
|
| 370 |
import marimo as mo
|
| 371 |
return (mo,)
|
| 372 |
|
| 373 |
|
| 374 |
+
@app.cell(disabled=True)
|
| 375 |
+
def _():
|
| 376 |
+
# You may need to install `fsspec ` and `adlfs ` beyond the dependencies included in the notebook
|
| 377 |
+
import os
|
| 378 |
+
import adlfs
|
| 379 |
+
return adlfs, os
|
| 380 |
+
|
| 381 |
+
|
| 382 |
@app.cell
|
| 383 |
def _():
|
| 384 |
import pathlib
|