Skip to main content
dual-attention
Projects
dual_attention--tiny_stories-LM
Workspace
Log in
Sign up
Project
Workspace
Runs
Automat.
Sweeps
Reports
Artifacts
Awni00's workspace
Personal workspace
Automated workspace
Changes are only visible to you.
Runs
48
Name
48 visualized
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_24_55
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_24_55
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_23_56
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_23_56
sa=4; ra=4; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_12_34_33
sa=4; ra=4; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_12_34_33
sa=6; ra=2; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_10_06_22
sa=6; ra=2; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_10_06_22
sa=4; ra=4; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_09_12_24
sa=4; ra=4; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_09_12_24
sa=6; ra=2; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_08_30_59
sa=6; ra=2; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_08_30_59
sa=4; ra=4; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_59_40
sa=4; ra=4; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_59_40
sa=6; ra=2; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_51_08
sa=6; ra=2; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_51_08
sa=6; ra=2; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_03_40
sa=6; ra=2; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_03_40
sa=6; ra=2; d=64; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_03_22_22
sa=6; ra=2; d=64; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_03_22_22
sa=4; ra=4; d=64; L=5; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_22_23_33
sa=4; ra=4; d=64; L=5; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_22_23_33
sa=4; ra=4; d=64; L=5; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_49
sa=4; ra=4; d=64; L=5; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_49
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_49
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_49
sa=6; ra=2; d=128; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=128; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=position_relative; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=False; symbol_type=position_relative; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=64; L=5; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=64; L=5; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_18_40_39
sa=6; ra=2; d=128; L=6; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_08_10_40
sa=6; ra=2; d=128; L=6; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_14_08_10_40
sa=8; d=128; L=6; pos_enc_type=RoPE__2024_05_14_08_10_35
sa=8; d=128; L=6; pos_enc_type=RoPE__2024_05_14_08_10_35
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_13_19_23_15
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_13_19_23_15
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=True; symbol_type=position_relative; pos_enc_type=RoPE__2024_05_13_19_23_15
sa=6; ra=2; d=64; L=4; ra_type=rel_attn; sym_rel=True; symbol_type=position_relative; pos_enc_type=RoPE__2024_05_13_19_23_15
1-20
of 48
mfu
mfu
Showing first 10 runs
0
20k
40k
60k
80k
100k
Step
-100
-80
-60
-40
-20
0
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=True; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_24_55
sa=4; ra=4; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_18_17_23_56
sa=4; ra=4; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_12_34_33
sa=6; ra=2; d=64; L=6; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_10_06_22
sa=4; ra=4; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_09_12_24
sa=6; ra=2; d=64; L=5; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_08_30_59
sa=4; ra=4; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_59_40
sa=6; ra=2; d=64; L=4; ra_type=rca; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_51_08
sa=6; ra=2; d=128; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_04_03_40
sa=6; ra=2; d=64; L=6; ra_type=rel_attn; sym_rel=False; symbol_type=symbolic_attention; pos_enc_type=RoPE__2024_05_15_03_22_22
Previous
Next