convert_to_jsonl.py (1320B)
1 import argparse 2 import datetime 3 import json 4 import pathlib 5 6 import datasets 7 8 9 def parse_command_argument() -> argparse.Namespace: 10 """ Parse command line arguments. """ 11 parser = argparse.ArgumentParser(description="Convert the huggingface dataset into jsonl files in the same director.") 12 parser.add_argument("dataset", type=pathlib.Path, help="Path to the input huggingface dataset.") 13 14 args: argparse.Namespace = parser.parse_args() 15 return args 16 17 18 def convert(x): 19 if isinstance(x, int) or isinstance(x, str): 20 return x 21 if isinstance(x, datetime.datetime): 22 return x.strftime("%Y-%m-%d %H:%M") 23 if isinstance(x, datetime.date): 24 return x.strftime("%Y-%m-%d") 25 raise RuntimeError("Unknown type") 26 27 28 def main(dataset_path: pathlib.Path) -> None: 29 splits: dict[str, datasets.Dataset] = {path.name: datasets.load_from_disk(str(path)) for path in dataset_path.iterdir() } 30 for name, split in splits.items(): 31 target: pathlib.Path = dataset_path / f"{name}.jsonl" 32 with target.open("w") as output: 33 for sample in split: 34 print(json.dumps({key: convert(value) for key, value in sample.items()}), file=output) 35 36 37 if __name__ == "__main__": 38 args: argparse.Namespace = parse_command_argument() 39 main(dataset_path=args.dataset)