prepare_data.sh (2673B)
1 #!/bin/sh 2 3 if [ -z "$DATA_PATH" ]; then 4 echo "Error: Variable DATA_PATH is not set." 5 echo "Please set it to the path where you want to store the data." 6 echo "For example, from the root of the git repository you can type:" 7 echo ' export DATA_PATH="$PWD/../data"' 8 exit 1 9 fi 10 11 if [ ! -f "$DATA_PATH/mmsrl/constraint-data.zip" ]; then 12 echo 'Data not found at $DATA_PATH/mmsrl/constraint-data.zip.' 13 echo "For information DATA_PATH is set to: $(readlink -m $DATA_PATH)" 14 echo "Downloading dataset..." 15 mkdir -p "$DATA_PATH/mmsrl" 16 curl "https://esimon.eu/tmp/constraint-data.zip" --output "$DATA_PATH/mmsrl/constraint-data.zip" 17 fi 18 19 if [ ! -f "$DATA_PATH/mmsrl/constraint22_dataset_unseen_test.zip" ]; then 20 echo 'Test data not found at $DATA_PATH/mmsrl/constraint22_dataset_unseen_test.zip.' 21 echo "For information DATA_PATH is set to: $(readlink -m $DATA_PATH)" 22 echo "Downloading test dataset..." 23 curl "https://esimon.eu/tmp/constraint22_dataset_unseen_test.zip" --output "$DATA_PATH/mmsrl/constraint22_dataset_unseen_test.zip" 24 fi 25 26 echo "Preparing data in $DATA_PATH" 27 echo "Unzipping" 28 unzip -q "$DATA_PATH/mmsrl/constraint-data.zip" -d "$DATA_PATH/mmsrl" 29 unzip -q "$DATA_PATH/mmsrl/constraint22_dataset_covid19.zip" -d "$DATA_PATH/mmsrl/covid19" 30 unzip -q "$DATA_PATH/mmsrl/constraint22_dataset_uspolitics.zip" -d "$DATA_PATH/mmsrl/uspolitics" 31 unzip -q "$DATA_PATH/mmsrl/constraint22_dataset_unseen_test.zip" -d "$DATA_PATH/mmsrl/all" 32 33 if [ ! -f "$DATA_PATH/mmsrl/all/annotations/test.jsonl" ]; then 34 echo 'Labeled test data not found at $DATA_PATH/mmsrl/all/annotations/test.jsonl.' 35 echo "For information DATA_PATH is set to: $(readlink -m $DATA_PATH)" 36 echo "Downloading test dataset..." 37 curl "https://esimon.eu/tmp/constraint22_dataset_unseen_test_gold_labels.jsonl" --output "$DATA_PATH/mmsrl/all/annotations/test.jsonl" 38 fi 39 40 echo "Merging datasets" 41 cat "$DATA_PATH/mmsrl/covid19/annotations/train.jsonl" "$DATA_PATH/mmsrl/uspolitics/annotations/train.jsonl" | grep -v '"hero": \[\], "villain": \[\], "victim": \[\], "other": \[\]' > "$DATA_PATH/mmsrl/all/annotations/train.jsonl" 42 cat "$DATA_PATH/mmsrl/covid19/annotations/val.jsonl" "$DATA_PATH/mmsrl/uspolitics/annotations/val.jsonl" | grep -v '"hero": \[\], "villain": \[\], "victim": \[\], "other": \[\]' > "$DATA_PATH/mmsrl/all/annotations/val.jsonl" 43 cp -rl "$DATA_PATH/mmsrl/covid19/images" "$DATA_PATH/mmsrl/all" 44 cp -rl "$DATA_PATH/mmsrl/uspolitics/images" "$DATA_PATH/mmsrl/all" 45 rm -r "$DATA_PATH/mmsrl/covid19/images" 46 rm -r "$DATA_PATH/mmsrl/uspolitics/images" 47 ln -s "$DATA_PATH/mmsrl/all/images" "$DATA_PATH/mmsrl/covid19/images" 48 ln -s "$DATA_PATH/mmsrl/all/images" "$DATA_PATH/mmsrl/uspolitics/images"