Downstream blocks can control the volume of data it loads into memory from its upstream block’s output.The downstream block will only load the chunk from indigo_mountain where power equals 5.
1
Load data
Copy
import randomimport polars as plfrom mage_ai.data.tabular.mocks import create_dataframe@data_loaderdef load_data(*args, **kwargs): dfs = [] for i in range(10): df = create_dataframe(n_rows=100_000, use_pandas=False) df = df.with_columns(pl.lit(i).alias('power')) if i == 5: df = df.with_columns(pl.lit(i).cast(pl.Float64).alias('col_0')) dfs.append(df) return pl.concat(dfs)
2
Transform data
Copy
@transformerdef transform(data, *args, **kwargs): return data