5m 9s

0.9

0.95

1.0000e-8

false

1000

false

false

0

false

4

false

false

true

false

1

1

false

false

false

mmap

/mnt/ssd-cluster/data/enron/enron_text_document

local

false

true

false

{"train_batch_size":192,"train_micro_batch_size_per_gpu":4,"gradient_accumulation_steps":8,"optimizer":{"type":"Adam","params":{"lr":0.0006,"max_grad_norm":1,"betas":[0.9,0.95]}},"fp16":{"fp16":true,"enabled":true,"loss_scale":0,"loss_scale_window":1000,"hysteresis":2,"min_loss_scale":1},"gradient_clipping":1,"zero_optimization":{"stage":0,"allgather_partitions":true,"allgather_bucket_size":500000000,"overlap_comm":true,"reduce_scatter":true,"reduce_bucket_size":500000000,"contiguous_gradients":true,"cpu_offload":false},"steps_per_print":10,"wall_clock_breakdown":true,"deepspeed":true}

false

false

nccl