To run the latest Gemma 4 model from Google at 31B parameters with full quantization using unsloth, follow the steps below:
{
"name": "gemma_4_31b_full_quantization",
"cells": [
{
"cell_type": "markdown",
"source": [
"# Requirements"
]
},
{
"cell_type": "code",
"source": [
"import torch\n",
"import os\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\n",
"torch.cuda.set_device(\"0\")\n",
"print(f'PyTorch version: {torch.__version__}')\n",
"print(f'GPU visible devices: {torch.cuda.device_count()}")
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Install Required Libraries"
]
},
{
"cell_type": "code",
"source": [
"!pip install --quiet pip\n",
"!pip install --quiet tqdm\n",
"!pip install --quiet git+/unsloth/unsloth.git\n",
"!pip install --quiet transformers==4.33\n",
"!pip install --quiet torchaudio\n",
"!pip install --quiet pyyaml\n"
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Load Model and Tokenizer"
]
},
{
"cell_type": "code",
"source": [
"from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
"model_name = \"google/gemini-15pro\""
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Set Up Model Parameters"
]
},
{
"cell_type": "code",
"source": [
"model_params = {\n",
" 'torch_dtype': torch.float16,\n",
" 'trust_remote_code': True,\n",
" 'device': \"cuda\",\n",
" 'num_train_tokens': 10000000000,\n",
" 'max_new_tokens': 10000,\n",
" 'do_sample': True,\n",
" 'temperature': 0.7,\n",
" 'repetition_penalty': 1.2,\n",
" 'penalty_temperature': 0.8,\n",
" 'penalty_strength': 0.6\n",
"}"
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Load Model and Tokenizer"
]
},
{
"cell_type": "code",
"source": [
"tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
"model = AutoModelForSequenceClassification.from_pretrained(model_name).to('cuda')\n",
"print(f'Model loaded successfully.')\n",
"print(f'Tokenizer loaded successfully.')"
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Generate Text"
]
},
{
"cell_type": "code",
"source": [
"def generate_text(prompt, max_length=100):\n",
" inputs = tokenizer(prompt, return_tensors=\"pt\", truncation=True, padding="max_length", max_length=max_length)\n",
" with torch.no_grad():\n",
" outputs = model(**inputs)\n",
" logits = outputs.logits\n",
" probabilities = torch.softmax(logits, dim=-1)\n",
" top_k = int(probabilities.topk(1)[1] * 100)\n",
" sampled_token = tokenizer.decode(torch.multinomial(probabilities, num_samples=top_k)).strip()\n",
" return sampled_token\n"
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Example Usage"
]
},
{
"cell_type": "code",
"source": [
"if __name__ == '__main__':\n",
" prompt = \"A quick brown fox jumps over the lazy dog.\" \n",
" generated_text = generate_text(prompt)\n",
" print(f'Generated text:\n{generated_text}\n')"
],
"execution_state": "running",
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# Output"
]
},
{
"cell_type": "code",
"source": [
"print(f'Generated text:\n{generated_text}\n')"
],
"execution_state": "running",
"outputs": []
}
]
}