Spaces:
Paused
Paused
File size: 3,264 Bytes
932a3f7 c9e3139 932a3f7 aa45833 2c7734a 28e546f 194a589 932a3f7 268a397 194a589 aa45833 932a3f7 28e546f 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 932a3f7 31cf774 aa45833 932a3f7 aa45833 d2ab08c 932a3f7 aa45833 932a3f7 aa45833 932a3f7 aa45833 621ccec 4c8b395 621ccec 932a3f7 aa45833 932a3f7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
FROM nvidia/cuda:12.5.1-cudnn-devel-ubuntu20.04
ENV DEBIAN_FRONTEND=noninteractive \
TZ=Asia/Kolkata \
PYTHONUNBUFFERED=1
# Install some basic utilities
RUN rm -f /etc/apt/sources.list.d/*.list && \
apt-get update && apt-get install -y --no-install-recommends \
curl \
ca-certificates \
sudo \
git \
wget \
procps \
git-lfs \
zip \
unzip \
htop \
vim \
nano \
bzip2 \
libx11-6 \
build-essential \
libsndfile-dev \
software-properties-common \
&& rm -rf /var/lib/apt/lists/*
RUN add-apt-repository ppa:flexiondotorg/nvtop && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends nvtop
# Create a working directory
WORKDIR /app
# Create a non-root user and switch to it
RUN adduser --disabled-password --gecos '' --shell /bin/bash user \
&& chown -R user:user /app
RUN echo "user ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-user
USER user
# All users can use /home/user as their home directory
ENV HOME=/home/user
RUN mkdir $HOME/.cache $HOME/.config \
&& chmod -R 777 $HOME
# Set up the Conda environment
ENV CONDA_AUTO_UPDATE_CONDA=false \
PATH=$HOME/miniconda/bin:$PATH
RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \
&& chmod +x ~/miniconda.sh \
&& ~/miniconda.sh -b -p ~/miniconda \
&& rm ~/miniconda.sh \
&& conda clean -ya
WORKDIR $HOME/app
#######################################
# Start root user section
#######################################
USER root
# User Debian packages
## Security warning : Potential user code executed as root (build time)
RUN --mount=target=/root/packages.txt,source=packages.txt \
apt-get update && \
xargs -r -a /root/packages.txt apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
RUN --mount=target=/root/on_startup.sh,source=on_startup.sh,readwrite \
bash /root/on_startup.sh
RUN mkdir /data && chown user:user /data
#######################################
# End root user section
#######################################
USER user
ENV VLLM_LOGGING_LEVEL=DEBUG
ENV LD_LIBRARY_PATH=/usr/local/cuda-12.5/compat:${LD_LIBRARY_PATH}
# # Python packages
# RUN --mount=target=requirements.txt,source=requirements.txt \
# pip install --no-cache-dir --upgrade -r requirements.txt
# Upgrade pip
RUN python3 -m pip install --upgrade pip
# Install CUDA-compatible torch first, then vLLM
RUN pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cu121 \
torch torchvision torchaudio && \
pip install --no-cache-dir vllm
# pip install vllm --torch-backend=auto
# Copy the current directory contents into the container at $HOME/app setting the owner to the user
COPY --chown=user . $HOME/app
RUN chmod +x start_server.sh
# Default env vars
ENV MODEL_NAME="unsloth/llama-2-7b-bnb-4bit" \
VLLM_PORT=7860 \
HOST=0.0.0.0 \
TP_SIZE=1 \
GPU_MEMORY_UTILIZATION=0.90
ENV PYTHONUNBUFFERED=1 \
GRADIO_ALLOW_FLAGGING=never \
GRADIO_NUM_PORTS=1 \
GRADIO_SERVER_NAME=0.0.0.0 \
GRADIO_THEME=huggingface \
SYSTEM=spaces \
SHELL=/bin/bash
RUN nvcc --version
RUN ls /usr/local | grep cuda
RUN find /usr/ -name 'libcuda.so.*'
CMD ["./start_server.sh"]
|