size_t elem_size = 0; switch(out->data_type) case 0x08: case 0x09: elem_size = 1; break; case 0x0B: elem_size = 2; break; case 0x0C: elem_size = 4; break; case 0x0D: elem_size = 4; break; case 0x0E: elem_size = 8; break; default: free(out->dims); fclose(f); return -5;
out->data_size_bytes = total_elements * elem_size; out->data = malloc(out->data_size_bytes); if (fread(out->data, 1, out->data_size_bytes, f) != out->data_size_bytes) free(out->dims); free(out->data); fclose(f); return -6; idx file
with open(filename, 'wb') as f: # Write magic: [0, 0, type_code, dim_count] f.write(bytes([0, 0, data_type_code, dim_count])) # Write dimensions (big-endian) for dim in data_array.shape: f.write(dim.to_bytes(4, 'big')) # Write data (row-major, native endianness) # Convert to flat bytes in correct order data_array.astype(data_array.dtype, copy=False).tofile(f) #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <arpa/inet.h> typedef struct idx_file uint8_t data_type; // 0x08,0x09,0x0B-0x0E uint8_t dim_count; // 1-255 uint32_t *dims; // array of dim_count sizes void *data; // raw data pointer size_t data_size_bytes; idx_file_t; size_t elem_size = 0
int idx_read(const char *filename, idx_file_t *out) header[1] != 0) fclose(f); return -3; // Invalid magic prefix case 0x0B: elem_size = 2
fclose(f); return 0; Benchmark results (average of 10 runs, reading 60k MNIST images):