vars.env

NCCL_P2P_DISABLE=1
GPU_MEMORY_UTILIZATION=0.7
#OMP_NUM_THREADS=4

# LiteLLM Configuration Defaults (can be overridden per-model in models.yml)
#LITELLM_TEMPERATURE=0.7
#LITELLM_TOP_P=0.8
#LITELLM_TOP_K=20
#LITELLM_REPETITION_PENALTY=1.05
#LITELLM_MAX_TOKENS=65536
#LITELLM_TIMEOUT=30

# IMPORTANT: Concurrency limits to prevent hanging requests from blocking others
# With vLLM --max-num-seqs 16, limit concurrent requests to avoid request pileups.
# Setting this too high causes requests to queue up behind slow/hung ones.
# A value of 5 ensures at most 5 concurrent requests, leaving room for fast
# requests to complete while slow ones timeout instead of stacking up.
#LITELLM_MAX_PARALLEL_REQUESTS=5