sdlaml-llm

Kessel's workspace

Runs

2,335

loss_metrics/global_avg_loss

loss_metrics/global_max_loss

lr/opt_0/group_0

lr/opt_0/group_1

lr/opt_0/group_2

memory/max_active(%)

memory/max_active(GiB)

memory/max_reserved(%)

memory/max_reserved(GiB)

memory/num_alloc_retries

memory/num_ooms

mfu(%)

tflops

throughput(tps)

time_metrics/data_loading(%)

time_metrics/data_loading(s)

time_metrics/end_to_end(s)

track_param_condition_number/model_part_0/layers.0.attention.wk

track_param_condition_number/model_part_0/layers.0.attention.wo

track_param_condition_number/model_part_0/layers.0.attention.wq

track_param_condition_number/model_part_0/layers.0.attention.wv

track_param_condition_number/model_part_0/layers.0.feed_forward.w1

track_param_condition_number/model_part_0/layers.0.feed_forward.w2

track_param_condition_number/model_part_0/layers.0.feed_forward.w3

track_param_condition_number/model_part_0/layers.1.attention.wk

track_param_condition_number/model_part_0/layers.1.attention.wo

track_param_condition_number/model_part_0/layers.1.attention.wq

track_param_condition_number/model_part_0/layers.1.attention.wv

track_param_condition_number/model_part_0/layers.1.feed_forward.w1

track_param_condition_number/model_part_0/layers.1.feed_forward.w2

track_param_condition_number/model_part_0/layers.1.feed_forward.w3

track_param_condition_number/model_part_0/layers.10.attention.wk

track_param_condition_number/model_part_0/layers.10.attention.wo

track_param_condition_number/model_part_0/layers.10.attention.wq

track_param_condition_number/model_part_0/layers.10.attention.wv

track_param_condition_number/model_part_0/layers.10.feed_forward.w1

track_param_condition_number/model_part_0/layers.10.feed_forward.w2

track_param_condition_number/model_part_0/layers.10.feed_forward.w3

track_param_condition_number/model_part_0/layers.100.attention.wk

track_param_condition_number/model_part_0/layers.100.attention.wo

track_param_condition_number/model_part_0/layers.100.attention.wq

track_param_condition_number/model_part_0/layers.100.attention.wv

track_param_condition_number/model_part_0/layers.100.feed_forward.w1

track_param_condition_number/model_part_0/layers.100.feed_forward.w2

Finished

ofivite

1mo ago

19s

4.12969

5.03049

0.70711

72.00524

28.43811

73.18561

28.9043

15.73344

49.08834

20249.48956

3.4818

0.028171

0.80911

21563.88281

4351.05322

85980.78906

581.68311

753.26068

747.01184

761.17084

2674.08936

1166.73425

1436.33203

18115.44336

646.90619

690.73535

637.26819

2651.9978

3271.31006

4184.54297

1283.18921

649.633

678.86121

696.03613

3168.36621

1070.45898

1524.93628

13163.39355

650.57227

1878.32056

Finished

ofivite

1mo ago

17s

4.29059

4.76435

0.70711

72.00524

28.43811

73.07682

28.86133

12.42898

38.77843

15996.53453

3.2466

0.033252

4.09689

15124.49902

40221.61328

12382.71191

10011.37988

328.84348

465.88019

315.82184

13821.28418

62086.66797

13638.45313

7618.0166

269.44272

379.73825

294.83069

25685.60156

33490.77734

32384.71289

19565.09961

228.81108

405.28619

247.40347

60832.03125

32860.34766

129340.5625

33113.96484

414.64615

1100.41565

Finished

ofivite

1mo ago

20s

4.2787

4.54174

0.70711

72.10453

28.47733

73.45266

29.00977

17.30984

54.0067

22278.36814

3.63497

0.026732

0.73542

15969.4209

646.60754

15819.94727

2322.72363

927.03833

1103.62866

883.16412

1322.90149

1761.02234

9282.63379

3190.22119

880.43512

1026.55688

863.10559

1159.96167

513.81177

877.21051

1998.04089

667.20593

818.67578

758.35052

9021.44629

3263.34888

7327.73096

1932.08728

789.37299

1967.67151

Finished

ofivite

1mo ago

17s

4.08447

5.20354

0.70711

72.00524

28.43811

73.07682

28.86133

12.03476

37.54846

15489.15966

3.13218

0.033131

2.11554

23230.36133

31579.0957

98251.92188

6259.46533

428.2319

545.84595

457.33456

5759.04492

14218.79883

2227.43848

18885.2793

400.97186

588.39679

428.2099

4124.44336

3796.64453

5614.00977

2519.97168

344.20108

506.43063

381.21408

282001.6875

23059.39648

52535.33984

79165.11719

412.39203

1306.09302

Finished

ofivite

1mo ago

16s

4.1219

5.83213

0.70711

71.95559

28.41851

73.07682

28.86133

11.4749

35.8017

14768.60119

2.91352

0.032322

1.10938

3853.09546

731.4563

15303.96387

6901.0708

528.80487

618.23035

550.0495

2627.64331

674.70258

3370.13135

1805.77332

448.16205

578.71326

460.76184

2271.74268

27287.22266

1692.58875

788.31958

478.70383

513.17212

487.76974

62422.60547

16965.1875

22134.38086

121273.67969

433.73566

1521.37659

Finished

ofivite

1mo ago

17s

4.94326

5.49311

0.70711

72.00524

28.43812

73.07682

28.86133

12.71547

39.67227

16365.2574

3.37603

0.033799

8.00916

29365.92969

6691.03271

32999.5625

30980.64453

226.07637

290.60153

233.69481

4132.5376

7864.13916

33502.53125

5386.71289

196.01967

331.70215

210.8885

19058.9707

32190.78906

48834.95703

3262.46436

146.95076

320.37665

164.15259

69378.65625

47088.81641

34583.85547

575552.0625

231.87151

823.78802

Finished

ofivite

1mo ago

17s

3.99687

4.87884

0.17678

72.00524

28.43811

73.18561

28.9043

15.94664

49.75353

20523.88638

3.58604

0.028627

0.79829

8871.36719

271.85889

306419.875

1075.76367

152.03525

378.93307

144.09776

7770.27734

847.34918

3204.41431

9994.79297

150.60434

441.06625

169.47031

2690.70752

1859.45532

52856.28906

997.72308

143.52956

448.83401

160.46092

51660.6875

2271.72192

10361.89844

10867.60352

196.73453

993.54175

Finished

ofivite

1mo ago

17s

4.20825

4.67946

0.17678

72.00524

28.43811

73.07682

28.86133

12.43482

38.79664

16004.05023

3.2097

0.032859

4.09496

159353.45313

719.25037

30695.93164

2584.92651

67.61385

226.22697

73.28381

9620.56445

2273.57129

6860.51758

19658.70117

68.34104

286.21063

68.72919

1119.82031

919.06763

8386.22168

9272.99707

60.63507

364.02106

65.54527

10358.43066

77302.49219

14057.8877

6098566.5

113.41306

1040.33362

Finished

ofivite

1mo ago

20s

4.1109

4.36574

0.17678

72.10453

28.47733

73.45266

29.00977

17.65809

55.09323

22726.5728

3.70761

0.026729

0.72092

11974.41602

3484.10889

5519.72754

985.64368

181.62047

503.54364

178.35719

3928.28638

589.69379

1942.48547

523.23688

191.26884

467.43188

172.58302

2169.86841

2853.32227

2526.15088

1278.89929

154.10367

464.97818

180.22304

6627.11914

1045.79114

1877.91919

2351.84595

227.7491

1019.43964

Finished

ofivite

1mo ago

17s

4.03215

5.06616

0.17678

72.00524

28.43811

73.07682

28.86133

12.14474

37.89159

15630.70599

3.12828

0.03279

2.09639

83914.35156

753.51117

5624.58203

1467.79785

77.62476

242.85716

92.39297

1268.06177

644.51099

2212.00757

835.5

92.66469

363.07565

99.90571

2049.31494

572.57471

2298.27637

843.11725

96.46036

493.70297

97.51347

91589.0625

26562.11523

81702

226035.45313

146.35036

1215.94556

Finished

ofivite

1mo ago

17s

4.00491

5.7688

0.17678

71.95559

28.41851

73.07682

28.86133

11.53079

35.97605

14840.52353

2.91683

0.032202

1.104

13052.29688

471.78583

8956.68164

1130.48486

106.36986

262.23477

113.78255

3688.65161

579.21338

1374.66589

3758.75928

123.56757

370.5018

114.93404

1034.29529

41976.92969

2237.85718

16252.16797

126.40884

467.20645

151.60919

41606.53906

12115.2373

34508.89063

6800.34033

182.83917

1256.70886

Finished

ofivite

1mo ago

16s

4.79769

5.33183

0.17678

72.00524

28.43812

73.07682

28.86133

12.68038

39.56279

16320.0951

3.45423

0.034678

8.03133

815273.25

2601.93359

19055.27344

1247.27625

51.2423

217.59541

53.773

10216.66797

7204.25244

12685.74219

3433.70361

44.17289

220.68732

46.3866

2569.70044

30571.83008

5831.59717

6359.21533

45.50513

319.86154

45.83411

6891.29346

72764.32813

28586.73438

16502.54297

78.73653

784.22101

Finished

ofivite

1mo ago

16s

3.91209

4.75941

0.044194

72.00524

28.43811

73.18561

28.9043

15.9781

49.85167

20564.37249

3.57835

0.028509

0.79672

9239.18262

4104.1416

8561.33301

6517.68652

6.73364

5.34042

4.68357

5846.84521

941.37231

7981.80469

6434.11914

18.95309

5.90451

11.18626

171702.5625

1206.92334

182524.875

1082.37659

23.86503

14.21678

15.97182

2970.88232

3860.75439

19617.57813

17899.36523

26.2914

29.71335

Finished

ofivite

1mo ago

16s

3.87609

4.36625

0.044194

72.00524

28.43811

73.07682

28.86133

12.6377

39.42962

16265.1604

3.35579

0.033803

4.02923

26704.39844

2582.96875

68981.28906

4849.53955

8.21917

6.39638

5.83693

2334.68433

463.51233

1406.48657

1254.43457

7.98691

8.21142

6.61035

4551.37549

8229.18457

9370.40332

3530.59741

15.20289

15.4437

11.00239

4010.01392

190215.51563

267167.03125

769278.625

15.03882

21.03493

Finished

ofivite

1mo ago

20s

4.06325

4.31291

0.044194

72.10453

28.47733

73.45266

29.00977

17.5495

54.75443

22586.81693

3.68387

0.026722

0.72538

4379.80127

4256.50977

7035.90674

2280.59595

6.05859

5.53391

4.32226

3119.05176

6605.32764

1674.59192

1271.41931

12.74932

6.10013

8.38727

21841.66602

31451.98828

17440.89648

1462.43689

31.74056

14.78366

22.71416

1952.63269

6746.57227

5452.76416

3063.45435

29.00676

20.30518

Finished

ofivite

1mo ago

18s

3.97954

5.00083

0.044194

72.00524

28.43811

73.07682

28.86133

12.21537

38.11194

15721.60332

3.1539

0.032868

2.08427

26476.20508

38722.78125

47453.42188

1669.67053

9.85656

10.0825

6.13612

9438.20117

1343.73926

6385.79834

1635.18982

10.66454

6.03595

7.98742

5328.57373

2045.5946

12775.35938

7877.72949

17.06084

20.00262

16.88357

9308.10547

19008.37305

80685.28125

278387.53125

32.19229

85.8576

Finished

ofivite

1mo ago

17s

3.95686

5.70224

0.044194

71.95559

28.41851

73.07682

28.86133

11.54389

36.01694

14857.39151

2.94007

0.032422

1.10275

18287.19531

18793.20703

8131.0918

3646.32129

8.40943

6.11476

5.51377

3164.7771

6050.104

5690.73486

6406.1499

11.76171

7.11864

7.9996

7199.04541

9909.75879

19861.74414

4434.32959

19.08951

12.48701

15.01989

6720.63818

8244.96191

4622.43799

59874.86328

35.73114

81.76465

Finished

ofivite

1mo ago

19s

4.38316

4.58964

0.044194

72.00524

28.43812

73.07682

28.86133

12.73978

39.74812

16396.5453

3.36375

0.033612

7.99388

7849.1333

2699.39819

64148.38281

3914.62305

5.69367

5.95163

5.08437

3322.12036

1836.0271

3486.32178

4419.13037

6.15167

7.49537

5.81249

2761.88574

1850.04041

5212.53516

12050.16699

10.46729

29.0225

10.56062

17800.25977

19459.78125

5330.36963

489758.625

20.318

152.03896

Finished

ofivite

1mo ago

16s

3.95238

4.78803

0.011049

72.00524

28.43811

73.18561

28.9043

16.11143

50.26765

20735.97031

3.62367

0.028631

0.79012

726.4931

3302.53101

11910.68359

2082.91553

4.29343

6.59281

4.53022

4949.79199

1916.76416

2029.65503

1652.9613

5.63224

6.03773

5.24147

150252.96875

9538.9082

130115.63281

3863.69849

18.07847

7.72517

18.92002

5176.76611

12213.00781

6590.07617

3819.15845

9.9212

5.21111

Finished

ofivite

1mo ago

16s

4.16233

4.61615

0.011049

72.00524

28.43811

73.07682

28.86133

12.6107

39.34539

16230.41433

3.36708

0.033989

4.03785

604.39154

6602.61523

1467.11963

21.00784

6.21085

7.9002

3.60582

494.56329

2796.39844

762.03442

25.28544

23.58293

5.78372

13.99023

3359.16675

2123.92847

1240.47827

9.65783

5.24561

6.68111

3.73959

354.85406

25896.32422

789.51202

60.07235

6.60664

5.1963

1-20

of 2,335