example_dataset_config_train.yaml

settings:
  src_path: data/raw/redpajama_v2_samples_512_train.jsonl
  dst_path: data/mem_map/redpajama_v2_samples_512_train.pbin
  index_path: data/mem_map/redpajama_v2_samples_512_train.idx
  jq_pattern: .raw_content
  num_cpus: ${node_env:num_cpus}
  eod_token: <|endoftext|>
  processing_batch_size: 1000
  raw_samples_queue_size: 300
  processed_samples_queue_size: 300

tokenizer:
  component_key: tokenizer
  variant_key: pretrained_hf_tokenizer
  config:
    pretrained_model_name_or_path: tokenizer
    padding: false
    truncation: false