milvus-io /
milvus
| @@ 199-221 (lines=23) @@ | ||
| 196 | ||
| 197 | print("Preparing dataset", dbname) |
|
| 198 | ||
| 199 | if dbname.startswith('SIFT'): |
|
| 200 | # SIFT1M to SIFT1000M |
|
| 201 | dbsize = int(dbname[4:-1]) |
|
| 202 | xb = mmap_bvecs('bigann/bigann_base.bvecs') |
|
| 203 | xq = mmap_bvecs('bigann/bigann_query.bvecs') |
|
| 204 | xt = mmap_bvecs('bigann/bigann_learn.bvecs') |
|
| 205 | ||
| 206 | # trim xb to correct size |
|
| 207 | xb = xb[:dbsize * 1000 * 1000] |
|
| 208 | ||
| 209 | gt_I = ivecs_read('bigann/gnd/idx_%dM.ivecs' % dbsize) |
|
| 210 | ||
| 211 | elif dbname == 'Deep1B': |
|
| 212 | xb = mmap_fvecs('deep1b/base.fvecs') |
|
| 213 | xq = mmap_fvecs('deep1b/deep1B_queries.fvecs') |
|
| 214 | xt = mmap_fvecs('deep1b/learn.fvecs') |
|
| 215 | # deep1B's train is is outrageously big |
|
| 216 | xt = xt[:10 * 1000 * 1000] |
|
| 217 | gt_I = ivecs_read('deep1b/deep1B_groundtruth.ivecs') |
|
| 218 | ||
| 219 | else: |
|
| 220 | print('unknown dataset', dbname, file=sys.stderr) |
|
| 221 | sys.exit(1) |
|
| 222 | ||
| 223 | ||
| 224 | if knngraph: |
|
| @@ 59-81 (lines=23) @@ | ||
| 56 | ||
| 57 | print("Preparing dataset", dbname) |
|
| 58 | ||
| 59 | if dbname.startswith('SIFT'): |
|
| 60 | # SIFT1M to SIFT1000M |
|
| 61 | dbsize = int(dbname[4:-1]) |
|
| 62 | xb = mmap_bvecs('bigann/bigann_base.bvecs') |
|
| 63 | xq = mmap_bvecs('bigann/bigann_query.bvecs') |
|
| 64 | xt = mmap_bvecs('bigann/bigann_learn.bvecs') |
|
| 65 | ||
| 66 | # trim xb to correct size |
|
| 67 | xb = xb[:dbsize * 1000 * 1000] |
|
| 68 | ||
| 69 | gt = ivecs_read('bigann/gnd/idx_%dM.ivecs' % dbsize) |
|
| 70 | ||
| 71 | elif dbname == 'Deep1B': |
|
| 72 | xb = mmap_fvecs('deep1b/base.fvecs') |
|
| 73 | xq = mmap_fvecs('deep1b/deep1B_queries.fvecs') |
|
| 74 | xt = mmap_fvecs('deep1b/learn.fvecs') |
|
| 75 | # deep1B's train is is outrageously big |
|
| 76 | xt = xt[:10 * 1000 * 1000] |
|
| 77 | gt = ivecs_read('deep1b/deep1B_groundtruth.ivecs') |
|
| 78 | ||
| 79 | else: |
|
| 80 | print('unknown dataset', dbname, file=sys.stderr) |
|
| 81 | sys.exit(1) |
|
| 82 | ||
| 83 | ||
| 84 | print("sizes: B %s Q %s T %s gt %s" % ( |
|
