Saving weights and logs of step 1200

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e17e7235a47f50d90b1e481dcb09e9727dfe9a345b6cc36b8cc1cfd6a583c81
+oid sha256:1370699db9ee8980b9d18ba78ab3c7bacbf64455af8b4d767abadaf4e1c6a466
 size 891548548

run_t5_mlm_flax_custom_dataset.py

@@ -703,6 +703,13 @@ if __name__ == "__main__":
     else:
         model = FlaxT5ForConditionalGeneration(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
 
+
+    # def to_bf16(t):
+    #     return jax.tree_map(lambda x: x.astype(jnp.bfloat16) if x.dtype == jnp.float32 else x, t)
+    #
+    #
+    # model.params = to_bf16(model.params)
+
     # Data collator
     # This one will take care of randomly masking the tokens.
     data_collator = FlaxDataCollatorForT5MLM(

runs/Jul10_08-38-10_t1v-n-0e7426e8-w-0/events.out.tfevents.1625906314.t1v-n-0e7426e8-w-0.25839.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1a444a03c97dd08a17796eec5b0c22674d96d4475fcb3dc8e47d8c3ec25db74
-size 136359
+oid sha256:b87fa89d0ac5eeabdea48a6a8250033be187061f5d0f1635b1d3f57ce6c7daaf
+size 181839

streaming_dataset_filter_test.py

@@ -0,0 +1,24 @@
+from clean import clean_text
+
+from datasets import load_dataset
+
+dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)
+
+
+def f(obj):
+    obj["text"] = clean_text(obj["text"])
+    return obj
+
+
+dataset_v1 = dataset_v0.map(f)
+it = iter(dataset_v0)
+
+print(next(it))
+print(next(it))
+print(next(it))
+
+it = iter(dataset_v1)
+
+print(next(it))
+print(next(it))
+print(next(it))