line_offsets_of_seekable_zstd.c (3263B)
1 #include <assert.h> 2 #include <stdbool.h> 3 #include <stdint.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <string.h> 7 8 #include "zstd-seek.h" 9 10 11 #ifndef NREAD 12 #define NREAD 100 13 #endif 14 15 16 bool isLittleEndian(){ 17 volatile int x = 1; 18 return *(char*)(&x) == 1; 19 } 20 21 void writeLE32(char **output, uint32_t data){ 22 if(isLittleEndian()){ 23 memcpy(*output, &data, sizeof(data)); 24 } else { 25 uint32_t swap = ((data & 0xFF000000) >> 24) | 26 ((data & 0x00FF0000) >> 8) | 27 ((data & 0x0000FF00) << 8) | 28 ((data & 0x000000FF) << 24); 29 memcpy(*output, &swap, sizeof(swap)); 30 } 31 *output += sizeof(uint32_t); 32 } 33 34 int main(int argc, char **argv){ 35 if(argc != 2){ 36 fprintf(stderr, "format: %s path/to/seekable.file.zstd\n", argv[0]); 37 return EXIT_FAILURE; 38 } 39 40 ZSTDSeek_Context *sctx = ZSTDSeek_createFromFile(argv[1]); 41 if(!sctx){ 42 fprintf(stderr, "Can't open zstd file.\n"); 43 return EXIT_FAILURE; 44 } 45 46 ZSTDSeek_JumpTable *jt = ZSTDSeek_getJumpTableOfContext(sctx); 47 if(!jt){ 48 fprintf(stderr, "Zstd file not seekable.\n"); 49 return EXIT_FAILURE; 50 } 51 52 size_t max_buffer = 0; 53 uint64_t last_uncompressed = 0; 54 for(ZSTDSeek_JumpTableRecord *record = jt->records; record != jt->records + jt->length; ++record){ 55 size_t candidate = record->uncompressedPos - last_uncompressed; 56 if(max_buffer < candidate) 57 max_buffer = candidate; 58 last_uncompressed = record->uncompressedPos; 59 } 60 61 size_t output_size = strlen(argv[1]) + 1 + sizeof(uint32_t)*(1+2*(jt->length-1)); 62 char *output_start = malloc(output_size); 63 char *output_end = output_start + output_size; 64 char *output = output_start; 65 66 strcpy(output, argv[1]); 67 output += strlen(argv[1]) + 1; 68 writeLE32(&output, jt->length-1); 69 70 char *buffer = malloc(max_buffer * NREAD); 71 for(ZSTDSeek_JumpTableRecord *brecord = jt->records; brecord < jt->records + jt->length - 1; brecord += NREAD){ 72 assert(ZSTDSeek_tell(sctx) == brecord->uncompressedPos); 73 assert(output < output_end); 74 ZSTDSeek_JumpTableRecord *end = brecord+NREAD<jt->records+jt->length-1?brecord+NREAD:jt->records+jt->length-1; 75 76 size_t read_size = 0; 77 for(ZSTDSeek_JumpTableRecord *record = brecord; record < end; ++record) 78 read_size += record[1].uncompressedPos - record->uncompressedPos; 79 size_t read = ZSTDSeek_read(buffer, read_size, sctx); 80 81 size_t buffer_offset = 0; 82 for(ZSTDSeek_JumpTableRecord *record = brecord; record < end; ++record){ 83 uint32_t compressed_size = record[1].compressedPos - record->compressedPos; 84 uint32_t uncompressed_size = record[1].uncompressedPos - record->uncompressedPos; 85 86 uint32_t newlines = 0; 87 for(size_t i = buffer_offset; i < buffer_offset+uncompressed_size; ++i) 88 if(buffer[i] == '\n') 89 ++newlines; 90 writeLE32(&output, newlines); 91 writeLE32(&output, compressed_size); 92 93 buffer_offset += uncompressed_size; 94 } 95 } 96 97 assert(output == output_end); 98 99 char *output_name = malloc(strlen(argv[1]) + sizeof(".zindex")); 100 strcpy(output_name, argv[1]); 101 strcat(output_name, ".zindex"); 102 103 FILE *output_file = fopen(output_name, "wb"); 104 size_t r = fwrite(output_start, output_size, 1, output_file); 105 fclose(output_file); 106 107 free(output_name); 108 free(buffer); 109 ZSTDSeek_free(sctx); 110 return EXIT_SUCCESS; 111 }