Level 99 dynamic abilities

@transformer
def transform(data, *args, **kwargs):
    print('Batch size:', len(data[0]))
    print('Chunks:', len(data))

Batch size: 143
Chunks: 21105

@transformer
def transform(data, *args, **kwargs):
    for batch in data:
        df = batch.deserialize()
        print(df.shape)

(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
(1337, 11)
...

from mage_ai.data.tabular.reader import (
    read_metadata,
    sample_batch_datasets,
    scan_batch_datasets_generator,
)

@transformer
def transform(data, *args, **kwargs):
    print(data.chunks)
    print(data.number_of_outputs)
    print(data.resource_usages)
    print(data.variable_path)
    print(data.variable_type)
    print(data.data_source)

    output = data.read_sync()
    for batch in output:
        df = batch.deserialize()
        print(df.shape)

import random

import polars as pl

from mage_ai.data.tabular.mocks import create_dataframe


@data_loader
def load_data(*args, **kwargs):
    dfs = []
    for i in range(10):
        df = create_dataframe(n_rows=100_000, use_pandas=False)
        df = df.with_columns(pl.lit(i).alias('power'))
        if i == 5:
            df = df.with_columns(pl.lit(i).cast(pl.Float64).alias('col_0'))
        dfs.append(df)
    return pl.concat(dfs)