| 123456789101112131415161718192021 |
- # @package _global_
- defaults:
- - /experiment/owt/gpt2l-flash.yaml
- - override /model/gpt2model: gpt2-xlarge
- # Can enable mlp_checkpoint_lvl to fit to A100 40GB
- # model:
- # config:
- # # mlp_checkpoint_lvl: ${eval:"[1] * 18 + [2] * 18"}
- # mlp_checkpoint_lvl: 1
- datamodule:
- batch_size: ${eval:"2 if ${train.gpu_mem} < 24 else (4 if ${train.gpu_mem} < 40 else (8 if ${train.gpu_mem} < 80 else 16))"}
- # With adamw-zero optimizer, on A100 40GB:
- # checkpoint_lvl=1, batch size = 4: mem 37GB, 4650ms / batch of 512 (285ms * 15 + 375ms * 1)
- # checkpoint_lvl=1, batch size = 8: mem 46GB, 4330ms / batch of 512 (530ms * 7 + 620ms * 1)
- # checkpoint_lvl=2, batch size = 8: mem 41GB, 4570ms / batch of 512 (560ms * 7 + 650ms * 1)
- # With adamw-apex-distributed optimizer:
- # checkpoint_lvl=1, batch size = 8: mem 41.5GB, 4500ms / batch of 512 (550ms * 7 + 650ms * 1)
- # checkpoint_lvl=1 for 24 layers and checkpoint_lvl=2 for 24 layers,
- # batch size = 8: mem 39GB, 4640ms / batch of 512 (565ms * 7 + 675ms * 1)
|
