文档中心

		

			#数据集位置,从cos上拉取获得
		

		

			DATA_PATH=/cfs/ti-deepspeed-demo/webtext/data.json
		

		

			#模型输出
		

		

			CHECKPOINT_PATH=./output
		

		

			 
		

		

			DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK
		

		

			 --master_addr $MASTER_ADDR --master_port $MASTER_PORT"
		

		

			#超参数设置
		

		

			python -m torch.distributed.launch $DISTRIBUTED_ARGS \
		

		

			 pretrain_gpt2.py \
		

		

			 --deepspeed \
		

		

			 --deepspeed_config ./scripts/debug-10B.json \
		

		

			 --model-parallel-size 1 \
		

		

			 --num-layers 50 \
		

		

			 --hidden-size 4096 \
		

		

			 --num-attention-heads 32 \
		

		

			 --batch-size 6 \
		

		

			 --seq-length 1024 \
		

		

			 --max-position-embeddings 1024 \
		

		

			 --train-iters 500000 \
		

		

			 --save $CHECKPOINT_PATH \
		

		

			 --train-data $DATA_PATH \
		

		

			 --lazy-loader \
		

		

			 --tokenizer-type GPT2BPETokenizer \
		

		

			 --split 949,50,1 \
		

		

			 --distributed-backend nccl \
		

		

			 --lr 0.00015 \
		

		

			 --lr-decay-style cosine \
		

		

			 --weight-decay 1e-2 \
		

		

			 --clip-grad 1.0 \
		

		

			 --warmup .01 \
		

		

			 --checkpoint-activations \
		

		

			 --log-interval 1 \
		

		

			 --save-interval 10000 \
		

		

			 --eval-interval 1000 \
		

		

			 --eval-iters 10 \
		

		

			 --fp16
		

		

			#! /bin/bash
		

		

			 
		

		

			# Runs the "345M" parameter model
		

		

			 
		

		

			GPUS_PER_NODE=8
		

		

			MASTER_ADDR=${1:-localhost}
		

		

			MASTER_PORT=${2:-6000}
		

		

			NNODES=${3:-1}
		

		

			NODE_RANK=${4:-0}
		

		

			WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
		

		

			 
		

		

			#数据集位置,从cos上拉取获得
		

		

			DATA_PATH=/cfs/ti-deepspeed-demo/datasets/db_text_document
		

		

			VOCAB_PATH=./gpt2-vocab.json
		

		

			MERGE_PATH=./gpt2-merges.txt
		

		

			CHECKPOINT_PATH=./output
		

		

			 
		

		

			DISTRIBUTED_ARGS="--nproc_per_node $GPUS_PER_NODE --nnodes $NNODES --node_rank $NODE_RANK 
		

		

			--master_addr $MASTER_ADDR --master_port $MASTER_PORT"
		

		

			 
		

		

			python -m torch.distributed.launch $DISTRIBUTED_ARGS \
		

		

			 pretrain_gpt2.py \
		

		

			 --deepspeed \
		

		

			 --deepspeed_config ./scripts/100B.json \
		

		

			 --model-parallel-size 8 \
		

		

			 --num-layers 480 \
		

		

			 --hidden-size 4096 \
		

		

			 --num-attention-heads 32 \
		

		

			 --batch-size 6 \
		

		

			 --seq-length 1024 \
		

		

			 --max-position-embeddings 1024 \
		

		

			 --train-iters 500000 \
		

		

			 --save $CHECKPOINT_PATH \
		

		

			 --data-path $DATA_PATH \
		

		

			 --vocab-file $VOCAB_PATH \
		

		

			 --merge-file $MERGE_PATH \
		

		

			 --tokenizer-type GPT2BPETokenizer \
		

		

			 --split 949,50,1 \
		

		

			 --distributed-backend nccl \
		

		

			 --lr 0.00015 \
		

		

			 --lr-decay-style cosine \
		

		

			 --weight-decay 1e-2 \
		

		

			 --clip-grad 1.0 \
		

		

			 --warmup .01 \
		

		

			 --log-interval 1 \
		

		

			 --save-interval 10000 \
		

		

			 --eval-interval 1000 \
		

		

			 --eval-iters 10 \
		

		

			 --fp16 \
		

		

			 --cpu-optimizer \