flash-attention/training/configs/experiment/owt/gpt2s.yaml

# @package _global_
defaults:
  - /experiment/owt/base.yaml
  - override /model: gpt2
  - override /model/gpt2model: gpt2-small

datamodule:
  batch_size: ${eval:"4 if ${train.gpu_mem} < 24 else (8 if ${train.gpu_mem} < 40 else 16)"}