dual_attention--tiny_stories-LM Table

Awni00's workspace

Runs

Finished

awni00

2y ago

21h 27m 30s

swiglu

false

0.9

0.95

false

128

2024_05_18_17_24_55

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE

10005120

1781120

true

out/sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_18_17_24_55

RoPE

disentangled_v2

512

sym_attn

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_18_17_24_55

Finished

awni00

2y ago

21h 32m 3s

swiglu

false

0.9

0.95

false

128

2024_05_18_17_23_56

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

10103424

1879424

true

out/sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_18_17_23_56

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_18_17_23_56

Finished

awni00

2y ago

5h 16m 19s

swiglu

false

128

0.9

0.95

false

2024_05_15_12_34_33

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4583616

455616

true

out/sa=4; rca=4; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_12_34_33

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_12_34_33

Finished

awni00

2y ago

4h 28m

swiglu

false

128

0.9

0.95

false

2024_05_15_10_06_22

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4586688

458688

true

out/sa=6; rca=2; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_10_06_22

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=6; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_10_06_22

Finished

awni00

2y ago

4h 43m 52s

swiglu

false

128

0.9

0.95

false

2024_05_15_09_12_24

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4519296

391296

true

out/sa=4; rca=4; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_09_12_24

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_09_12_24

Finished

awni00

2y ago

4h 3m 21s

swiglu

false

128

0.9

0.95

false

2024_05_15_08_30_59

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4521856

393856

true

out/sa=6; rca=2; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_08_30_59

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=5; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_08_30_59

Finished

awni00

2y ago

4h 12m 30s

swiglu

false

128

0.9

0.95

false

2024_05_15_04_59_40

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4454976

326976

true

out/sa=4; rca=4; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_59_40

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_59_40

Finished

awni00

2y ago

3h 39m 39s

swiglu

false

128

0.9

0.95

false

2024_05_15_04_51_08

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4457024

329024

true

out/sa=6; rca=2; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_51_08

RoPE

standard

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=4; rca_type=standard; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_51_08

Finished

awni00

2y ago

14h 9m 49s

swiglu

false

128

0.9

0.95

false

128

2024_05_15_04_03_40

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

10114560

1890560

true

out/sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_03_40

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_04_03_40

Finished

awni00

2y ago

11h 56m 57s

swiglu

false

128

0.9

0.95

false

2024_05_15_03_22_22

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4636032

508032

true

out/sa=6; rca=2; d=64; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_03_22_22

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=6; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_15_03_22_22

Finished

awni00

2y ago

15h 24m 39s

swiglu

false

128

0.9

0.95

false

2024_05_14_22_23_33

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE

4540416

412416

true

out/sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_22_23_33

RoPE

disentangled_v2

512

sym_attn

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_22_23_33

Finished

awni00

2y ago

15h 25m 9s

swiglu

false

128

0.9

0.95

false

2024_05_14_18_40_49

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4560896

432896

true

out/sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_49

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=4; rca=4; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_49

Finished

awni00

2y ago

8h 41m 7s

swiglu

false

128

0.9

0.95

false

2024_05_14_18_40_49

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4489920

361920

true

out/sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_49

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_49

Finished

awni00

2y ago

10h 10m 12s

swiglu

false

128

0.9

0.95

false

128

2024_05_14_18_40_39

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=128; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

9533568

1309568

true

out/sa=6; rca=2; d=128; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_39

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=128; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_39

Finished

awni00

2y ago

9h 21m 38s

swiglu

false

128

0.9

0.95

false

2024_05_14_18_40_39

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=pos_relative; pos_enc_type=RoPE

4485824

357824

true

out/sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=pos_relative; pos_enc_type=RoPE__2024_05_14_18_40_39

RoPE

disentangled_v2

512

pos_relative

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=False; symbol_type=pos_relative; pos_enc_type=RoPE__2024_05_14_18_40_39

Finished

awni00

2y ago

10h 18m 45s

swiglu

false

128

0.9

0.95

false

2024_05_14_18_40_39

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE

4562976

434976

true

out/sa=6; rca=2; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_39

RoPE

disentangled_v2

512

sym_attn

false

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=5; rca_type=disentangled_v2; sym_rel=False; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_18_40_39

Finished

awni00

2y ago

14h 12m 38s

swiglu

false

128

0.9

0.95

false

128

2024_05_14_08_10_40

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE

10016256

1792256

true

out/sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_08_10_40

RoPE

disentangled_v2

512

sym_attn

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=128; L=6; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_14_08_10_40

Finished

awni00

2y ago

3h 37m 3s

swiglu

false

128

0.9

0.95

false

128

2024_05_14_08_10_35

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=8; d=128; L=6; pos_enc_type=RoPE

9799936

1575936

true

out/sa=8; d=128; L=6; pos_enc_type=RoPE__2024_05_14_08_10_35

RoPE

512

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=8; d=128; L=6; pos_enc_type=RoPE__2024_05_14_08_10_35

Finished

awni00

2y ago

8h 39m 32s

swiglu

false

128

0.9

0.95

false

2024_05_13_19_23_15

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE

4473536

345536

true

out/sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_13_19_23_15

RoPE

disentangled_v2

512

sym_attn

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=sym_attn; pos_enc_type=RoPE__2024_05_13_19_23_15

Finished

awni00

2y ago

9h 23m 34s

swiglu

false

128

0.9

0.95

false

2024_05_13_19_23_15

true

cuda

0.1

bfloat16

2000

100

false

scratch

0.001

100000

512

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=pos_relative; pos_enc_type=RoPE

4469440

341440

true

out/sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=pos_relative; pos_enc_type=RoPE__2024_05_13_19_23_15

RoPE

disentangled_v2

512

pos_relative

true

false

32000

llama2

true

abstract_transformer--tiny_stories-LM

sa=6; rca=2; d=64; L=4; rca_type=disentangled_v2; sym_rel=True; symbol_type=pos_relative; pos_enc_type=RoPE__2024_05_13_19_23_15

1-20

of 48