Inference_llms.ipynb (4169B)
1 { 2 "cells": [ 3 { 4 "cell_type": "code", 5 "execution_count": null, 6 "id": "d54d056e-1e7c-4cf5-ac36-e5f2cb8e7675", 7 "metadata": {}, 8 "outputs": [], 9 "source": [ 10 "from unsloth import FastLanguageModel\n", 11 "import torch\n", 12 "import pandas as pd \n", 13 "import re\n", 14 "import json\n", 15 "max_seq_length = 2048 \n", 16 "dtype = torch.float16\n", 17 "load_in_4bit = False \n" 18 ] 19 }, 20 { 21 "cell_type": "code", 22 "execution_count": null, 23 "id": "bfb56038-f8b4-4d5c-929d-eb3930c67bfa", 24 "metadata": {}, 25 "outputs": [], 26 "source": [ 27 "from unsloth import FastLanguageModel\n", 28 "model, tokenizer = FastLanguageModel.from_pretrained(\n", 29 " model_name = \"/cluster/work/users/anonymous/NProStrt_2_Llama-3.2-1B_128\", # YOUR MODEL YOU USED FOR TRAINING\n", 30 " max_seq_length = max_seq_length,\n", 31 " dtype = dtype,\n", 32 " load_in_4bit = False,\n", 33 ")\n", 34 "FastLanguageModel.for_inference(model)" 35 ] 36 }, 37 { 38 "cell_type": "code", 39 "execution_count": null, 40 "id": "07b9206e-6fb7-4b4c-b925-ea4859343f7b", 41 "metadata": {}, 42 "outputs": [], 43 "source": [ 44 "alpaca_prompt = \"\"\"\n", 45 "### Instruction:\n", 46 "{}\n", 47 "\n", 48 "### Input:\n", 49 "{}\n", 50 "\n", 51 "### Response:\n", 52 "{}\"\"\"\n", 53 "\n", 54 "\n", 55 "\n", 56 "inputss = tokenizer(\n", 57 "[\n", 58 " alpaca_prompt.format(\n", 59 " \"Extract the information in JSONL format for the following source_text:\", # instruction\n", 60 " \"Al-Hawl camp | Two women killed in south-eastern Al-Hasakah countryside SOHR sources have reported a new murder took place in Al-Hawl “mini-state” in the far south-eastern countryside of Al-Hasakah, when a displaced Syrian woman and unidentified woman were killed. According to SOHR sources, Internal Security Forces “Asayish” found the body of two women in the third section of the camp. This comes in light of ongoing attacks carried out by ISIS cells in Al-Hawl camp, which houses ISIS families and members. Accordingly, the number of murders documented in the camp since early 2022 has risen to 19, which left 20 people dead; and they are as follows: • Six Iraqi refugees, including two women. • Seven Syrian people: Two men and five women. • Six unidentified women. • A medic was killed in the post where he worked in the camp.\", # input\n", 61 " \"\", # output - leave this blank for generation!\n", 62 " )\n", 63 "], return_tensors = \"pt\").to(\"cuda\")" 64 ] 65 }, 66 { 67 "cell_type": "code", 68 "execution_count": null, 69 "id": "3060ea0b-fbed-4e0b-ac00-1b55364d1c0c", 70 "metadata": {}, 71 "outputs": [], 72 "source": [ 73 "dataFrame = pd.read_json(\"/cluster/work/users/anonymous/new_jsons_prompt/test_data_json_prompt2.json\")\n", 74 "list_out = []\n", 75 "for index, row in dataFrame.iterrows():\n", 76 " inputs = tokenizer(\n", 77 " [\n", 78 " alpaca_prompt.format(\n", 79 " str(row['instruction']), # instruction\n", 80 " str(row['input'][0:2000]), # input\n", 81 " \"\", # output - leave this blank for generation!\n", 82 " )\n", 83 " ], return_tensors = \"pt\").to(\"cuda\")\n", 84 "\n", 85 " outputs = model.generate(**inputs, max_new_tokens = 200, use_cache = True)\n", 86 " response_text = tokenizer.batch_decode(outputs)\n", 87 " list_out.append(\"id: \"+str(row['id'])+\" \"+ \"\".join( response_text))\n", 88 " print(\"id: \"+str(row['id'])+\" \"+ \"\".join( response_text))\n", 89 "print(list_out[0])\n", 90 "with open('output_NProStrt_2_Llama-3.2-1B_128.txt', 'w') as f:\n", 91 " for line in list_out:\n", 92 " f.write(f\"{line}\\n\\t\")" 93 ] 94 } 95 ], 96 "metadata": { 97 "kernelspec": { 98 "display_name": "Python 3 (ipykernel)", 99 "language": "python", 100 "name": "python3" 101 }, 102 "language_info": { 103 "codemirror_mode": { 104 "name": "ipython", 105 "version": 3 106 }, 107 "file_extension": ".py", 108 "mimetype": "text/x-python", 109 "name": "python", 110 "nbconvert_exporter": "python", 111 "pygments_lexer": "ipython3", 112 "version": "3.12.9" 113 } 114 }, 115 "nbformat": 4, 116 "nbformat_minor": 5 117 }