Aflah's workspace
Runs
160
Name
8 visualized
seq_length: 1024
seq_length: 1024
1
32
seq_length: 2048
seq_length: 2048
2
128
dataset: FW_Edu
dataset: FW_Edu
2
100
model: pythia
model: pythia
2
68
pos_emb: none
pos_emb: none
1
40
rotary_pct: 1
rotary_pct: 1
4
40
seed: 1234
seed: 1234
3
24
lr: 0.0004
lr: 0.0004
2
16
use_qk_layernorm: true
use_qk_layernorm: true
1
4
use_qk_layernorm: false
use_qk_layernorm: false
2
12
pipe_parallel_size: 0
pipe_parallel_size: 0
2
8
log_grad_norm: false
log_grad_norm: false
4
log_grad_norm: true
log_grad_norm: true
4
pipe_parallel_size: 1
pipe_parallel_size: 1
1
4
log_grad_norm: false
log_grad_norm: false
4
lr: 0.00004
lr: 0.00004
1
4
lr: 0.004
lr: 0.004
1
4
seed: 238
seed: 238
1
8
seed: 953
seed: 953
1
4
seed: 175
seed: 175
1
4
pos_emb: rotary
pos_emb: rotary
7
28
model: llama
model: llama
2
32
dataset: FW
dataset: FW
1
28
State
Notes
User
Tags
Created
Runtime
Sweep
activation
adlr_autoresume
adlr_autoresume_interval
allow_chopped
apply_query_key_layer_scaling
attention_config
attention_dropout
attention_softmax_in_fp32
batch_size
bf16.enabled
bias_dropout_fusion
bias_gelu_fusion
char_level_ppl
checkpoint_activations
checkpoint_factor
checkpoint_in_cpu
checkpoint_num_layers
checkpoint_scale
checkpoint_validation_with_forward_pass
clip_grad
config_files.1_1B_BS_32.yml
config_files.1_1B_BS_32_No_TE.yml
config_files.1_1B_BS_48.yml
config_files.1_1B_BS_48_No_TE.yml
config_files.1_1B_BS_48_No_TE_With_Softmax_Fusion.yml
config_files.1_1B_BS_48_With_Softmax_Fusion.yml
contiguous_checkpointing
coord_check
curriculum_seqlen
data_impl
data_path
data_types.grad_accum_dtype
dataset_impl
deepscale
deepspeed
deepspeed_activation_checkpointing
deepspeed_mpi
deepspeed_slurm
distributed_backend
dpo_beta
dpo_fp32
dpo_reference_free
dump_state
dynamic_loss_scale
Crashed
Finished
aflah
2d 10h 2m 44s
-
swiglu
false
1000
true
false
flash
0
false
128
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Failed
Finished
aflah
12d 4h 11m 7s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
["/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_100B_Merged_text_document","/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document"]
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Failed
Finished
aflah
12d 4h 11m 7s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Failed
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 55m 17s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
8d 16h 21m 55s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
1d 2h 27m 18s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 11m 45s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 20m 14s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 13m 47s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 13m 47s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 10m 37s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 3m 46s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Finished
aflah
17h 44m 26s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 12m 58s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
12h 18m 4s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Failed
Finished
aflah
1d 4h 20m 42s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Crashed
Finished
aflah
3d 19h 23m 37s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_Edu_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
Finished
aflah
2d 3h 39m 35s
-
swiglu
false
1000
true
false
flash
0
false
64
true
false
false
false
true
500
false
1
linear
false
1
-
-
-
-
-
-
false
false
0
mmap
/u/afkhan/Artifacts/RoPE_Pct/Datasets/FineWeb_100B_Merged_text_document
fp32
gpt2
false
true
true
false
false
nccl
0.1
true
false
false
true
1-2
of 2