Text Generation InferenceTGI (Text Generation Inference)

model=meta-llama/Llama-3.1-8B-Instruct
volume=$PWD/data

docker run --gpus all   -p 8080:80   -v $volume:/data   ghcr.io/huggingface/text-generation-inference:latest   --model-id $model   --num-shard 2   --max-input-length 4096   --max-total-tokens 8192   --quantize bitsandbytes-nf4

from huggingface_hub import InferenceClient

client = InferenceClient("http://localhost:8080")

# 기본 생성
response = client.text_generation(
    "한국의 수도는 어디인가요?",
    max_new_tokens=200,
    temperature=0.7,
    do_sample=True,
)
print(response)

# 스트리밍
for token in client.text_generation(
    "Python 비동기 프로그래밍을 설명해줘",
    max_new_tokens=500,
    stream=True,
):
    print(token, end="", flush=True)