Annotate HPLT Alignment.ipynb (14746B)
1 { 2 "cells": [ 3 { 4 "cell_type": "markdown", 5 "id": "295e2da7-45f4-4343-b2ec-75a07d137dcb", 6 "metadata": {}, 7 "source": [ 8 "# Annotate HPLT Alignment\n", 9 "\n", 10 "Small notebook to compare original UCDP documents to the one found in HPLT." 11 ] 12 }, 13 { 14 "cell_type": "code", 15 "id": "77ed1585-50cd-46ff-98de-7e4fbdb72c87", 16 "metadata": { 17 "jupyter": { 18 "source_hidden": true 19 }, 20 "ExecuteTime": { 21 "end_time": "2025-02-14T11:40:17.917514Z", 22 "start_time": "2025-02-14T11:40:17.834103Z" 23 } 24 }, 25 "source": [ 26 "%gui asyncio\n", 27 "\n", 28 "from __future__ import annotations\n", 29 "from typing import Any\n", 30 "import html\n", 31 "import json\n", 32 "import pathlib\n", 33 "import random\n", 34 "import subprocess\n", 35 "import tempfile\n", 36 "import urllib\n", 37 "\n", 38 "from IPython.display import DisplayHandle, display, update_display\n", 39 "import ipywidgets as widgets" 40 ], 41 "outputs": [], 42 "execution_count": 1 43 }, 44 { 45 "cell_type": "code", 46 "id": "4a9920eb-9c02-4a01-a7a1-9b4225568341", 47 "metadata": { 48 "jupyter": { 49 "source_hidden": true 50 }, 51 "ExecuteTime": { 52 "end_time": "2025-02-14T11:40:25.207085Z", 53 "start_time": "2025-02-14T11:40:25.200943Z" 54 } 55 }, 56 "source": [ 57 "DATA_PATH = \"https://recurrent.network/AEC/annotate-me.jsonl\"\n", 58 "ANNOTATIONS_PATH = pathlib.Path(\"./align_annotations.jsonl\")" 59 ], 60 "outputs": [], 61 "execution_count": 2 62 }, 63 { 64 "cell_type": "markdown", 65 "id": "00714869-a452-4963-a002-87f99fdf6714", 66 "metadata": {}, 67 "source": [ 68 "##### Data Loading" 69 ] 70 }, 71 { 72 "cell_type": "code", 73 "id": "92854922-b2b2-48bc-a2d3-c784dec04bb2", 74 "metadata": { 75 "jupyter": { 76 "source_hidden": true 77 }, 78 "ExecuteTime": { 79 "end_time": "2025-02-14T11:40:32.126357Z", 80 "start_time": "2025-02-14T11:40:28.190297Z" 81 } 82 }, 83 "source": [ 84 "print(\"(it'll take a minute or two) Retrieving the dataset… \", flush=True, end=\"\")\n", 85 "dataset: dict[int, dict[str, Any]] = {}\n", 86 "with urllib.request.urlopen(DATA_PATH) as data_file:\n", 87 " for line in data_file:\n", 88 " sample = json.loads(line)\n", 89 " dataset[sample[\"id\"]] = sample\n", 90 "print(\"done\")" 91 ], 92 "outputs": [ 93 { 94 "name": "stdout", 95 "output_type": "stream", 96 "text": [ 97 "(it'll take a minute or two) Retrieving the dataset… done\n" 98 ] 99 } 100 ], 101 "execution_count": 3 102 }, 103 { 104 "cell_type": "markdown", 105 "id": "9c030cde-f568-4c40-a8b6-7fa113204562", 106 "metadata": {}, 107 "source": [ 108 "## Annotations" 109 ] 110 }, 111 { 112 "cell_type": "code", 113 "id": "affa7a2e-abf5-49e0-81ea-f8939aad22d5", 114 "metadata": { 115 "jupyter": { 116 "source_hidden": true 117 }, 118 "ExecuteTime": { 119 "end_time": "2025-02-14T11:40:47.350351Z", 120 "start_time": "2025-02-14T11:40:47.306513Z" 121 } 122 }, 123 "source": [ 124 "class Annotator:\n", 125 " def __init__(self: Annotator) -> None:\n", 126 " self.handle: DisplayHandle = display(widgets.HTML(\"<p>Loading…</p>\"), display_id=True)\n", 127 " self.display_annotator_name_form()\n", 128 " self.init_display_buttons()\n", 129 "\n", 130 " def read_annotations(self: Annotator) -> None:\n", 131 " self.annotations: dict[int, dict[str, Any]] = {}\n", 132 " try:\n", 133 " with ANNOTATIONS_PATH.open(\"r\") as file:\n", 134 " for line in file:\n", 135 " annotation: dict[str, Any] = json.loads(line)\n", 136 " self.annotations[annotation[\"id\"]] = annotation\n", 137 " except FileNotFoundError:\n", 138 " raise RuntimeError(\"Annotation file empty, that's not normal! Are you runing jupyter from the directory containing the notebook and the annotations?\")\n", 139 "\n", 140 " def display_annotator_name_form(self: Annotator) -> None:\n", 141 " self.annotator_name_field = widgets.Text(placeholder=\"Your name\")\n", 142 " button = widgets.Button(description=\"Start\")\n", 143 " button.on_click(self.start_annotating)\n", 144 " self.read_annotations()\n", 145 " self.handle.update(widgets.VBox([\n", 146 " widgets.HTML(f\"Annotation file {ANNOTATIONS_PATH} contains {len(self.annotations)} annotation{'s' if len(self.annotations)>1 else ''}. Don't forget to push.\"),\n", 147 " widgets.HBox([widgets.HTML(\"Annotator's name: \"), self.annotator_name_field, button])\n", 148 " ]))\n", 149 "\n", 150 " def start_annotating(self: Annotator, _: widgets.Button) -> None:\n", 151 " self.handle.update(widgets.HTML(\"<p>Loading sample…</p>\"))\n", 152 " self.annotator_name: str = self.annotator_name_field.value\n", 153 " self.to_annotate: None | int = None\n", 154 " self.next_sample()\n", 155 "\n", 156 " def init_display_buttons(self: Annotator) -> None:\n", 157 " self.actor_checkbox = widgets.widgets.Checkbox(value=False, description=\"Wrong Actor\")\n", 158 " self.date_checkbox = widgets.widgets.Checkbox(value=False, description=\"Wrong Date\")\n", 159 " self.location_checkbox = widgets.widgets.Checkbox(value=False, description=\"Wrong Location\")\n", 160 " self.deaths_checkbox = widgets.widgets.Checkbox(value=False, description=\"Wrong Deaths\")\n", 161 " self.correct_button = widgets.Button(description=\"Similar Documents\", layout=widgets.Layout(width=\"250px\"))\n", 162 " self.mismatch_button = widgets.Button(description=\"Completly Different\", layout=widgets.Layout(width=\"250px\"))\n", 163 " self.comment_box = widgets.Text(placeholder=\"Comments\")\n", 164 "\n", 165 " self.correct_button.on_click(self.on_click)\n", 166 " self.mismatch_button.on_click(self.on_click)\n", 167 " self.annotation_buttons = widgets.HBox([self.actor_checkbox, self.date_checkbox, self.location_checkbox, self.deaths_checkbox, self.correct_button, self.mismatch_button, self.comment_box])\n", 168 "\n", 169 " def on_click(self: Annotator, target: widgets.Button) -> None:\n", 170 " similar = None\n", 171 " if target is self.correct_button:\n", 172 " similar = True\n", 173 " elif target is self.mismatch_button:\n", 174 " similar = False\n", 175 "\n", 176 " annotation: dict[str, Any] = {\n", 177 " \"annotator\": self.annotator_name,\n", 178 " \"id\": self.to_annotate,\n", 179 " \"similar\": similar,\n", 180 " \"wrong_actor\": self.actor_checkbox.value,\n", 181 " \"wrong_date\": self.date_checkbox.value,\n", 182 " \"wrong_location\": self.location_checkbox.value,\n", 183 " \"wrong_deaths\": self.deaths_checkbox.value,\n", 184 " \"comments\": self.comment_box.value,\n", 185 " } \n", 186 " with ANNOTATIONS_PATH.open(\"a\") as annotation_file:\n", 187 " print(json.dumps(annotation), file=annotation_file)\n", 188 " self.annotations[self.to_annotate] = annotation\n", 189 " \n", 190 " self.actor_checkbox.value = False\n", 191 " self.date_checkbox.value = False\n", 192 " self.location_checkbox.value = False\n", 193 " self.deaths_checkbox.value = False\n", 194 " self.comment_box.value = \"\"\n", 195 " self.next_sample()\n", 196 "\n", 197 " def next_sample(self: Annotator) -> None:\n", 198 " self.read_annotations()\n", 199 " candidates: set[int] = set(dataset.keys()) - self.annotations.keys()\n", 200 " self.to_annotate: int = random.choice(tuple(candidates))\n", 201 " sample: dict[str, Any] = dataset[self.to_annotate]\n", 202 " \n", 203 " ucdp_path = tempfile.NamedTemporaryFile()\n", 204 " hplt_path = tempfile.NamedTemporaryFile()\n", 205 " with tempfile.NamedTemporaryFile(delete_on_close=False, mode=\"w\") as ucdp_file, tempfile.NamedTemporaryFile(delete_on_close=False, mode=\"w\") as hplt_file:\n", 206 " print(sample[\"source_date\"], file=ucdp_file, end=\"\\n\")\n", 207 " print(sample[\"source_article_ucdp\"], file=ucdp_file, end=\"\")\n", 208 " print(sample[\"source_date\"], file=hplt_file, end=\"\\n\")\n", 209 " print(sample[\"source_article\"], file=hplt_file, end=\"\")\n", 210 " ucdp_file.close()\n", 211 " hplt_file.close()\n", 212 " result = subprocess.run([\"git\", \"diff\", \"--word-diff=porcelain\", ucdp_file.name, hplt_file.name], stdout=subprocess.PIPE, encoding=\"utf-8\")\n", 213 " \n", 214 " differences: list[tuple[str, str]] = []\n", 215 " header: bool = True\n", 216 " for line in result.stdout.split('\\n'):\n", 217 " if header:\n", 218 " if line and line[0] == \"@\":\n", 219 " header = False\n", 220 " continue\n", 221 " if not line:\n", 222 " continue\n", 223 " if line[0] == \"~\":\n", 224 " differences.append((\" \", \"<br>\"))\n", 225 " else:\n", 226 " text = html.escape(line[1:]).replace('$', \"$\")\n", 227 " if line[0] == \"+\":\n", 228 " text = f\"<ins>{text}</ins>\"\n", 229 " elif line[0] == \"-\":\n", 230 " text = f\"<del>{text}</del>\"\n", 231 " differences.append((line[0], text))\n", 232 " merged_diff: str = \"\".join(text for state, text in differences)\n", 233 " ucdp_diff: str = \"\".join(text for state, text in differences if state != '+')\n", 234 " hplt_diff: str = \"\".join(text for state, text in differences if state != '-')\n", 235 " \n", 236 " html_diff = f\"\"\"\n", 237 " <section>\n", 238 " <h3>Event #{self.to_annotate}</h3>\n", 239 " <style>\n", 240 " ins {{ background-color: #afa; text-decoration: none; }}\n", 241 " del {{ background-color: #fcc; text-decoration: none; }}\n", 242 " table.diff {{ width: 100%; }}\n", 243 " td, th {{ width: 50%; text-align: start !important; vertical-align: top !important; }}\n", 244 " </style>\n", 245 " \n", 246 " <section>\n", 247 " <h4>Merged Diff</h4>\n", 248 " <p>{merged_diff}</p>\n", 249 " </section>\n", 250 " <section>\n", 251 " <h4>Split Diff</h4>\n", 252 " <table class=\"\">\n", 253 " <tr><th>UCDP</th><th>HPLT</th></tr>\n", 254 " <tr><td>{ucdp_diff}</td><td>{hplt_diff}</td></tr>\n", 255 " </table>\n", 256 " </section>\n", 257 " <section>\n", 258 " <h4>UCDP Event Annotations</h4>\n", 259 " <table>\n", 260 " <tr><th>Side A</th><td>{sample['side_a_name']}</td></tr>\n", 261 " <tr><th>Side B</th><td>{sample['side_b_name']}</td></tr>\n", 262 " <tr><th>Start Date</th><td>{sample['start_date']}</td></tr>\n", 263 " <tr><th>End Date</th><td>{sample['end_date']}</td></tr>\n", 264 " <tr><th>Location Country</th><td>{sample['location_root_name']}</td></tr>\n", 265 " <tr><th>Location ADM1</th><td>{sample['location_adm1_name']}</td></tr>\n", 266 " <tr><th>Location ADM2</th><td>{sample['location_adm2_name']}</td></tr>\n", 267 " <tr><th>Location Where</th><td>{sample['location_where_name']}</td></tr>\n", 268 " <tr><th>Deaths Side A</th><td>{sample['deaths_side_a']}</td></tr>\n", 269 " <tr><th>Deaths Side B</th><td>{sample['deaths_side_b']}</td></tr>\n", 270 " <tr><th>Deaths Civilian</th><td>{sample['deaths_civilian']}</td></tr>\n", 271 " <tr><th>Deaths Unknown</th><td>{sample['deaths_unknown']}</td></tr>\n", 272 " <tr><th>Deaths Low</th><td>{sample['deaths_low']}</td></tr>\n", 273 " <tr><th>Deaths High</th><td>{sample['deaths_high']}</td></tr>\n", 274 " </table>\n", 275 " </section>\n", 276 " </section>\n", 277 " \"\"\"\n", 278 " self.handle.update(widgets.VBox([widgets.HTML(html_diff), self.annotation_buttons]))" 279 ], 280 "outputs": [], 281 "execution_count": 4 282 }, 283 { 284 "cell_type": "markdown", 285 "id": "24821340-b450-461a-88d1-70a68c3852db", 286 "metadata": {}, 287 "source": [ 288 "### Procedure\n", 289 "\n", 290 "If the articles are somewhat similar, but some pieces of information are missing or are wrong, check the boxes corresponding to the information that is wrong/missing:\n", 291 "- Who (actors) \n", 292 "- When (date)\n", 293 "- Where (location)\n", 294 "- What (number of deaths)\n", 295 "If the articles are completly different, you can click \"Completly Different\" without selecting the checkboxes.\n", 296 "If the articles are the same you can click \"Similar Documents\".\n", 297 "\n", 298 "If it's a complicated case you'd like to discuss, write a comment and post a message on Mattermost.\n", 299 "\n", 300 "**Don't forget to commit the `align_annotations.jsonl` file every once in a while.** It's saved and re-read after every annotation you do." 301 ] 302 }, 303 { 304 "cell_type": "code", 305 "id": "75f222af-33d5-41b8-9b7b-d476de57e504", 306 "metadata": { 307 "jupyter": { 308 "source_hidden": true 309 }, 310 "ExecuteTime": { 311 "end_time": "2025-02-14T11:40:54.618958Z", 312 "start_time": "2025-02-14T11:40:54.558133Z" 313 } 314 }, 315 "source": [ 316 "annotator = Annotator()" 317 ], 318 "outputs": [ 319 { 320 "data": { 321 "text/plain": [ 322 "VBox(children=(HTML(value='\\n <section>\\n <h3>Event #235759</h3>\\n <style>\\n …" 323 ], 324 "application/vnd.jupyter.widget-view+json": { 325 "version_major": 2, 326 "version_minor": 0, 327 "model_id": "2e2b5be596e74562aa4e36e131199482" 328 } 329 }, 330 "metadata": {}, 331 "output_type": "display_data" 332 } 333 ], 334 "execution_count": 5 335 } 336 ], 337 "metadata": { 338 "kernelspec": { 339 "display_name": "Python 3 (ipykernel)", 340 "language": "python", 341 "name": "python3" 342 }, 343 "language_info": { 344 "codemirror_mode": { 345 "name": "ipython", 346 "version": 3 347 }, 348 "file_extension": ".py", 349 "mimetype": "text/x-python", 350 "name": "python", 351 "nbconvert_exporter": "python", 352 "pygments_lexer": "ipython3", 353 "version": "3.12.8" 354 } 355 }, 356 "nbformat": 4, 357 "nbformat_minor": 5 358 }