streaming_dataset_filter_test.py

from clean import clean_text

from datasets import load_dataset

dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)


def f(obj):
    obj["text"] = clean_text(obj["text"])
    return obj


dataset_v1 = dataset_v0.map(f)
dataset_v2 = dataset_v1.filter(lambda obj: obj['text'] is not None)

it = iter(dataset_v0)
print(next(it))
print(next(it))
print(next(it))

it = iter(dataset_v1)
print(next(it))
print(next(it))
print(next(it))

it = iter(dataset_v2)
print(next(it))
print(next(it))
print(next(it))