run_retrieval1.sh (392B)
1 PKL=$1 2 HPLT_DIR=$2 3 OUTD=$3 4 5 TEXTFIELD=$4 # 'text', 't' for outputs from stage2 6 FILEPATTERN=$5 # '*.zst', 'text.zst' for outputs from stage2 7 8 echo "Running retrieval:" "$FILEPATTERN" 9 10 mkdir -p $OUTD 11 find $HPLT_DIR -name "$FILEPATTERN" | parallel --eta --joblog $OUTD/joblog --linebuffer -j 100 "python -u retriever.py retrieve {} -t $TEXTFIELD --fcontent=$PKL >$OUTD/{#}.out 2>$OUTD/{#}.err"