data_prep_FT.ipynb (17070B)
1 { 2 "cells": [ 3 { 4 "cell_type": "code", 5 "execution_count": 1, 6 "id": "63ef1545-c592-4664-a284-16c460523da6", 7 "metadata": {}, 8 "outputs": [], 9 "source": [ 10 "from datasets import load_from_disk\n", 11 "import pandas as pd\n", 12 "import json\n" 13 ] 14 }, 15 { 16 "cell_type": "code", 17 "execution_count": 2, 18 "id": "fb5affa5-b4f7-4a7c-a9d6-8a1c9d399eb6", 19 "metadata": {}, 20 "outputs": [], 21 "source": [ 22 "def load_data(path):\n", 23 " dataset_train = load_from_disk(path + \"/train\")\n", 24 " dataset_test = load_from_disk(path + \"/test\")\n", 25 " dataset_val = load_from_disk(path + \"/validation\")\n", 26 " train_data = pd.DataFrame(dataset_train)\n", 27 " val_data = pd.DataFrame(dataset_val)\n", 28 " test_data = pd.DataFrame(dataset_test)\n", 29 " \n", 30 " return train_data, val_data, test_data" 31 ] 32 }, 33 { 34 "cell_type": "code", 35 "execution_count": 49, 36 "id": "9a99c0b9-20ad-453b-837c-3458063aedbb", 37 "metadata": {}, 38 "outputs": [], 39 "source": [ 40 "def format_add_instruction(path, ins_type):\n", 41 " train_data, val_data, test_data = load_data(path)\n", 42 " train_data[\"start_date\"] = pd.to_datetime(train_data[\"start_date\"]).dt.strftime('%Y-%m-%d') \n", 43 " train_data[\"end_date\"] = pd.to_datetime(train_data[\"end_date\"]).dt.strftime('%Y-%m-%d')\n", 44 " val_data[\"start_date\"] = pd.to_datetime(val_data[\"start_date\"]).dt.strftime('%Y-%m-%d') \n", 45 " val_data[\"end_date\"] = pd.to_datetime(val_data[\"end_date\"]).dt.strftime('%Y-%m-%d')\n", 46 " test_data[\"start_date\"] = pd.to_datetime(test_data[\"start_date\"]).dt.strftime('%Y-%m-%d') \n", 47 " test_data[\"end_date\"] = pd.to_datetime(test_data[\"end_date\"]).dt.strftime('%Y-%m-%d') \n", 48 " if ins_type == 0:\n", 49 " train_data['Instruction']= \"Source UCDP document\"\n", 50 " val_data['Instruction']= \"Source UCDP document\"\n", 51 " test_data['Instruction']= \"Source UCDP document\"\n", 52 " elif ins_type == 1:\n", 53 " train_data['Instruction']= \"Generate the correct UCDP event information in JSON format for the incident of armed conflict described in the following source UCDP document:\"\n", 54 " val_data['Instruction']= \"Generate the correct UCDP event information in JSON format for the incident of armed conflict described in the following source UCDP document:\"\n", 55 " test_data['Instruction']= \"Generate the correct UCDP event information in JSON format for the incident of armed conflict described in the following source UCDP document:\"\n", 56 " elif ins_type == 2:\n", 57 " train_data['Instruction']= \"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {side_a_name, side_b_name, start_date, end_date, location_root_name, location_adm1_name, location_adm2_name, location_where_name, deaths_side_a, deaths_side_b, deaths_civilian, deaths_unknown, deaths_low, deaths_high}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"\n", 58 " val_data['Instruction']= \"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {side_a_name, side_b_name, start_date, end_date, location_root_name, location_adm1_name, location_adm2_name, location_where_name, deaths_side_a, deaths_side_b, deaths_civilian, deaths_unknown, deaths_low, deaths_high}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"\n", 59 " test_data['Instruction']= \"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {side_a_name, side_b_name, start_date, end_date, location_root_name, location_adm1_name, location_adm2_name, location_where_name, deaths_side_a, deaths_side_b, deaths_civilian, deaths_unknown, deaths_low, deaths_high}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"\n", 60 " elif ins_type == 3:\n", 61 " train_data['Instruction']= '\"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {\"side_a_name\": \"Actor side A, a party involved in the conflict, which may be a government of a state or an organized non-governmental group.\", \"side_b_name\": \"Actor side B, a party involved in the conflict, which may be a government of a state, an organized non-governmental group, or civilians.\", \"start_date\": \"The earliest likely date when the event has taken place.\", \"end_date\": \"The last likely date when the event has taken place.\", \"location_root_name\": \"The country where the event took place.\", \"location_adm1_name\": \"The region where the event took place.\", \"location_adm2_name\": \"The sub-region where the event took place.\", \"location_where_name\": \"A fine-grained description of the location where the event took place.\", \"deaths_side_a\": \"The best estimate of fatalities in Actor side A.\", \"deaths_side_b\": \"The best estimate of fatalities in Actor side B.\", \"deaths_civilian\": \"Number of civilian fatalities.\", \"deaths_unknown\": \"The best estimate of fatalities of persons of uncertain affiliation.\", \"deaths_low\": \"The most conservative estimate of deaths identified in the source text.\", \"deaths_high\": \"The highest reliable estimate of deaths identified in the source text.\"}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"'\n", 62 " val_data['Instruction']= '\"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {\"side_a_name\": \"Actor side A, a party involved in the conflict, which may be a government of a state or an organized non-governmental group.\", \"side_b_name\": \"Actor side B, a party involved in the conflict, which may be a government of a state, an organized non-governmental group, or civilians.\", \"start_date\": \"The earliest likely date when the event has taken place.\", \"end_date\": \"The last likely date when the event has taken place.\", \"location_root_name\": \"The country where the event took place.\", \"location_adm1_name\": \"The region where the event took place.\", \"location_adm2_name\": \"The sub-region where the event took place.\", \"location_where_name\": \"A fine-grained description of the location where the event took place.\", \"deaths_side_a\": \"The best estimate of fatalities in Actor side A.\", \"deaths_side_b\": \"The best estimate of fatalities in Actor side B.\", \"deaths_civilian\": \"Number of civilian fatalities.\", \"deaths_unknown\": \"The best estimate of fatalities of persons of uncertain affiliation.\", \"deaths_low\": \"The most conservative estimate of deaths identified in the source text.\", \"deaths_high\": \"The highest reliable estimate of deaths identified in the source text.\"}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"'\n", 63 " test_data['Instruction']= '\"Generate structured event information related to a socio-political conflict in the UCDP event format, based on the content of a source UCDP document. The generated data should include the following fields: {\"side_a_name\": \"Actor side A, a party involved in the conflict, which may be a government of a state or an organized non-governmental group.\", \"side_b_name\": \"Actor side B, a party involved in the conflict, which may be a government of a state, an organized non-governmental group, or civilians.\", \"start_date\": \"The earliest likely date when the event has taken place.\", \"end_date\": \"The last likely date when the event has taken place.\", \"location_root_name\": \"The country where the event took place.\", \"location_adm1_name\": \"The region where the event took place.\", \"location_adm2_name\": \"The sub-region where the event took place.\", \"location_where_name\": \"A fine-grained description of the location where the event took place.\", \"deaths_side_a\": \"The best estimate of fatalities in Actor side A.\", \"deaths_side_b\": \"The best estimate of fatalities in Actor side B.\", \"deaths_civilian\": \"Number of civilian fatalities.\", \"deaths_unknown\": \"The best estimate of fatalities of persons of uncertain affiliation.\", \"deaths_low\": \"The most conservative estimate of deaths identified in the source text.\", \"deaths_high\": \"The highest reliable estimate of deaths identified in the source text.\"}. Output the event information as a valid JSON object corresponding to the armed conflict incident described in the following source UCDP document:\"'\n", 64 " \n", 65 " \n", 66 " return train_data, val_data, test_data\n", 67 " " 68 ] 69 }, 70 { 71 "cell_type": "code", 72 "execution_count": 50, 73 "id": "24c8e60b-4d6e-4abb-a947-63c0fe26e5eb", 74 "metadata": {}, 75 "outputs": [], 76 "source": [ 77 "def add_ouput(_data):\n", 78 " selected_columns = [\"side_a_name\", \"side_b_name\", \"start_date\", \"end_date\", \"location_root_name\", \"location_adm1_name\", \"location_adm2_name\", \"location_where_name\", \"deaths_side_a\", \"deaths_side_b\", \"deaths_civilian\", \"deaths_unknown\" , \"deaths_low\", \"deaths_high\"] \n", 79 " \n", 80 " Output = []\n", 81 " for index, row in _data[selected_columns].iterrows():\n", 82 " row_json = row.to_json() # Convert row to JSON\n", 83 " Output.append(row_json) \n", 84 " _data['Output'] = Output\n", 85 " return _data\n", 86 " " 87 ] 88 }, 89 { 90 "cell_type": "code", 91 "execution_count": 28, 92 "id": "b53ec159-98df-42ce-920c-f2d293c4322c", 93 "metadata": {}, 94 "outputs": [], 95 "source": [ 96 "# header = [\"id\", \"source_article\", \"Instruction\", \"Output\"]\n", 97 "# train_data.to_csv('/Users/anonymous/Downloads/train_data_woi_js.csv', columns = header)\n", 98 "# val_data.to_csv('/Users/anonymous/Downloads/val_data_woi_js.csv', columns = header)\n", 99 "# test_data.to_csv('/Users/anonymous/Downloads/test_data_woi_js.csv', columns = header)\n" 100 ] 101 }, 102 { 103 "cell_type": "code", 104 "execution_count": 72, 105 "id": "5b68cd32-deb3-438c-bb5a-439b6a230825", 106 "metadata": {}, 107 "outputs": [], 108 "source": [ 109 "ins_type = 0\n", 110 "path = \"/Users/anonymous/Downloads/UCDP-AEC\"\n", 111 "train_data, val_data, test_data = format_add_instruction(path, ins_type)\n", 112 "train_data = add_ouput(train_data)\n", 113 "val_data = add_ouput(val_data)\n", 114 "test_data = add_ouput(test_data)" 115 ] 116 }, 117 { 118 "cell_type": "code", 119 "execution_count": 73, 120 "id": "db0ecb97-1754-48ca-a1ca-3eb9375e60b7", 121 "metadata": {}, 122 "outputs": [], 123 "source": [ 124 "def create_json(data_json):\n", 125 " dataset_json = [\n", 126 " { \"id\": row_dict[\"id\"],\n", 127 " \"instruction\": row_dict[\"Instruction\"],\n", 128 " \"input\": str(row_dict[\"source_date\"])+ \" \"+row_dict[\"source_article\"],\n", 129 " \"output\": row_dict[\"Output\"]\n", 130 " }\n", 131 " for row_dict in data_json.to_dict(orient=\"records\")\n", 132 " ]\n", 133 " return dataset_json" 134 ] 135 }, 136 { 137 "cell_type": "code", 138 "execution_count": 74, 139 "id": "a186c40f-80ae-4df1-a0b5-bfc6fc8b4609", 140 "metadata": {}, 141 "outputs": [], 142 "source": [ 143 "train_data_json = create_json(train_data)\n", 144 "test_data_json = create_json(test_data)\n", 145 "val_data_json = create_json(val_data)" 146 ] 147 }, 148 { 149 "cell_type": "code", 150 "execution_count": 75, 151 "id": "e6c599a8-739c-47ba-a86b-1165fe4e4e36", 152 "metadata": {}, 153 "outputs": [ 154 { 155 "data": { 156 "text/plain": [ 157 "{'id': 416729,\n", 158 " 'instruction': 'Source UCDP document',\n", 159 " 'input': '2021-09-17 00:00:00 Anglophone Separatists Ambush, Kill Many Cameroon Soldiers\\nSoldiers of the Cameroon army also killed three separatist fighters, including a self-proclaimed General.\\nA yet to be determined number of Cameroonian soldiers were killed in an ambush by Anglophone separatists in Ngoketunjia division of the Northwest region.\\nThe soldiers were killed on Thursday, Sept. 16, when their armoured car hit a landmine and burst into flames, killing some of its occupants while some who jumped from the burning vehicle were murdered by separatist fighters.\\nThe military is yet to issue any statement about this incident but separatists have posted videos of the incident on social media and are reporting that their fighters killed about 20 soldiers of the elite military unit of the Cameroonian army known as BIR.\\nIn a separate encounter, three separatist fighters including a self-proclaimed “general” were killed on Wednesday, Sept. 15, by the Cameroon military in Nake Bokoko, a village in Mbonge sub-division of the Mbonge division in the Southwest region.\\nThe three separatists killed were identified as Commander Cross, Emanu Cigar and Gazola. They were killed in their sleep.\\nTwo other separatist combatants who were in the same neighbourhood escaped. Reliable sources in the village say the military raid on the village was a hunt for separatist fighters who have been operating along the Kumba-Mbonge road.\\nThe wives of the two separatists who succeeded in escaping were arrested by the army but were later released.\\nThe Kumba-Mbonge stretch of road in the Southwest region is one of the most dangerous highways in the region as it is infested by suspected separatist fighters who waylay travellers and extort money and valuables from them.\\nThe separatists are said to have imposed themselves as the authority charged with settling issues involving villagers, with inhumane treatment and fines meted on them.\\nVillagers also confirmed the confiscation of their farms, properties, and other important items when caught by Commander Cross violating any of their orders.\\nSupport Our Journalism\\nThere are millions of ordinary people affected by conflict in Africa whose stories are missing in the mainstream media. HumAngle is determined to tell those challenging and under-reported stories, hoping that the people impacted by these conflicts will find the safety and security they deserve.\\nTo ensure that we continue to provide public service coverage, we have a small favour to ask you. We want you to be part of our journalistic endeavour by contributing a token to us.\\nYour donation will further promote a robust, free, and independent media.Donate Here',\n", 160 " 'output': '{\"side_a_name\":\"Government of Cameroon\",\"side_b_name\":\"Ambazonia insurgents\",\"start_date\":\"2021-09-15\",\"end_date\":\"2021-09-15\",\"location_root_name\":\"Cameroon\",\"location_adm1_name\":\"South-West region\",\"location_adm2_name\":\"Meme d\\\\u00e9partement\",\"location_where_name\":\"Nake Bongwana village\",\"deaths_side_a\":0,\"deaths_side_b\":3,\"deaths_civilian\":0,\"deaths_unknown\":0,\"deaths_low\":3,\"deaths_high\":3}'}" 161 ] 162 }, 163 "execution_count": 75, 164 "metadata": {}, 165 "output_type": "execute_result" 166 } 167 ], 168 "source": [ 169 "val_data_json[1]" 170 ] 171 }, 172 { 173 "cell_type": "code", 174 "execution_count": 76, 175 "id": "a85aa50c-58e6-4ceb-8c27-1544647d28e3", 176 "metadata": {}, 177 "outputs": [], 178 "source": [ 179 "with open(\"train_data_json_prompt\"+str(ins_type)+\".json\", \"w\") as f:\n", 180 " json.dump(train_data_json, f)\n", 181 "with open(\"test_data_json_prompt\"+str(ins_type)+\".json\", \"w\") as f:\n", 182 " json.dump(test_data_json, f)\n", 183 "with open(\"val_data_json_prompt\"+str(ins_type)+\".json\", \"w\") as f:\n", 184 " json.dump(val_data_json, f)" 185 ] 186 }, 187 { 188 "cell_type": "code", 189 "execution_count": null, 190 "id": "c16a2b9b-978e-4b16-a3cb-a5667b4fea05", 191 "metadata": {}, 192 "outputs": [], 193 "source": [] 194 } 195 ], 196 "metadata": { 197 "kernelspec": { 198 "display_name": "Python 3 (ipykernel)", 199 "language": "python", 200 "name": "python3" 201 }, 202 "language_info": { 203 "codemirror_mode": { 204 "name": "ipython", 205 "version": 3 206 }, 207 "file_extension": ".py", 208 "mimetype": "text/x-python", 209 "name": "python", 210 "nbconvert_exporter": "python", 211 "pygments_lexer": "ipython3", 212 "version": "3.12.9" 213 } 214 }, 215 "nbformat": 4, 216 "nbformat_minor": 5 217 }