Hash :
d50d3db6
Author :
Date :
2021-01-07T05:43:30
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
/*
* Copyright (C) the libgit2 contributors. All rights reserved.
*
* This file is part of libgit2, distributed under the GNU GPL v2 with
* a Linking Exception. For full terms see the included COPYING file.
*/
#include "midx.h"
#include "buffer.h"
#include "futils.h"
#include "hash.h"
#include "odb.h"
#include "pack.h"
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
#define MIDX_VERSION 1
#define MIDX_OBJECT_ID_VERSION 1
struct git_midx_header {
uint32_t signature;
uint8_t version;
uint8_t object_id_version;
uint8_t chunks;
uint8_t base_midx_files;
uint32_t packfiles;
};
#define MIDX_PACKFILE_NAMES_ID 0x504e414d /* "PNAM" */
#define MIDX_OID_FANOUT_ID 0x4f494446 /* "OIDF" */
#define MIDX_OID_LOOKUP_ID 0x4f49444c /* "OIDL" */
#define MIDX_OBJECT_OFFSETS_ID 0x4f4f4646 /* "OOFF" */
#define MIDX_OBJECT_LARGE_OFFSETS_ID 0x4c4f4646 /* "LOFF" */
struct git_midx_chunk {
off64_t offset;
size_t length;
};
static int midx_error(const char *message)
{
git_error_set(GIT_ERROR_ODB, "invalid multi-pack-index file - %s", message);
return -1;
}
static int midx_parse_packfile_names(
git_midx_file *idx,
const unsigned char *data,
uint32_t packfiles,
struct git_midx_chunk *chunk)
{
int error;
uint32_t i;
char *packfile_name = (char *)(data + chunk->offset);
size_t chunk_size = chunk->length, len;
if (chunk->offset == 0)
return midx_error("missing Packfile Names chunk");
if (chunk->length == 0)
return midx_error("empty Packfile Names chunk");
if ((error = git_vector_init(&idx->packfile_names, packfiles, git__strcmp_cb)) < 0)
return error;
for (i = 0; i < packfiles; ++i) {
len = p_strnlen(packfile_name, chunk_size);
if (len == 0)
return midx_error("empty packfile name");
if (len + 1 > chunk_size)
return midx_error("unterminated packfile name");
git_vector_insert(&idx->packfile_names, packfile_name);
if (i && strcmp(git_vector_get(&idx->packfile_names, i - 1), packfile_name) >= 0)
return midx_error("packfile names are not sorted");
if (strlen(packfile_name) <= strlen(".idx") || git__suffixcmp(packfile_name, ".idx") != 0)
return midx_error("non-.idx packfile name");
if (strchr(packfile_name, '/') != NULL || strchr(packfile_name, '\\') != NULL)
return midx_error("non-local packfile");
packfile_name += len + 1;
chunk_size -= len + 1;
}
return 0;
}
static int midx_parse_oid_fanout(
git_midx_file *idx,
const unsigned char *data,
struct git_midx_chunk *chunk_oid_fanout)
{
uint32_t i, nr;
if (chunk_oid_fanout->offset == 0)
return midx_error("missing OID Fanout chunk");
if (chunk_oid_fanout->length == 0)
return midx_error("empty OID Fanout chunk");
if (chunk_oid_fanout->length != 256 * 4)
return midx_error("OID Fanout chunk has wrong length");
idx->oid_fanout = (const uint32_t *)(data + chunk_oid_fanout->offset);
nr = 0;
for (i = 0; i < 256; ++i) {
uint32_t n = ntohl(idx->oid_fanout[i]);
if (n < nr)
return midx_error("index is non-monotonic");
nr = n;
}
idx->num_objects = nr;
return 0;
}
static int midx_parse_oid_lookup(
git_midx_file *idx,
const unsigned char *data,
struct git_midx_chunk *chunk_oid_lookup)
{
uint32_t i;
git_oid *oid, *prev_oid, zero_oid = {{0}};
if (chunk_oid_lookup->offset == 0)
return midx_error("missing OID Lookup chunk");
if (chunk_oid_lookup->length == 0)
return midx_error("empty OID Lookup chunk");
if (chunk_oid_lookup->length != idx->num_objects * GIT_OID_RAWSZ)
return midx_error("OID Lookup chunk has wrong length");
idx->oid_lookup = oid = (git_oid *)(data + chunk_oid_lookup->offset);
prev_oid = &zero_oid;
for (i = 0; i < idx->num_objects; ++i, ++oid) {
if (git_oid_cmp(prev_oid, oid) >= 0)
return midx_error("OID Lookup index is non-monotonic");
prev_oid = oid;
}
return 0;
}
static int midx_parse_object_offsets(
git_midx_file *idx,
const unsigned char *data,
struct git_midx_chunk *chunk_object_offsets)
{
if (chunk_object_offsets->offset == 0)
return midx_error("missing Object Offsets chunk");
if (chunk_object_offsets->length == 0)
return midx_error("empty Object Offsets chunk");
if (chunk_object_offsets->length != idx->num_objects * 8)
return midx_error("Object Offsets chunk has wrong length");
idx->object_offsets = data + chunk_object_offsets->offset;
return 0;
}
static int midx_parse_object_large_offsets(
git_midx_file *idx,
const unsigned char *data,
struct git_midx_chunk *chunk_object_large_offsets)
{
if (chunk_object_large_offsets->length == 0)
return 0;
if (chunk_object_large_offsets->length % 8 != 0)
return midx_error("malformed Object Large Offsets chunk");
idx->object_large_offsets = data + chunk_object_large_offsets->offset;
idx->num_object_large_offsets = chunk_object_large_offsets->length / 8;
return 0;
}
int git_midx_parse(
git_midx_file *idx,
const unsigned char *data,
size_t size)
{
struct git_midx_header *hdr;
const unsigned char *chunk_hdr;
struct git_midx_chunk *last_chunk;
uint32_t i;
off64_t last_chunk_offset, chunk_offset, trailer_offset;
git_oid idx_checksum = {{0}};
int error;
struct git_midx_chunk chunk_packfile_names = {0},
chunk_oid_fanout = {0},
chunk_oid_lookup = {0},
chunk_object_offsets = {0},
chunk_object_large_offsets = {0};
GIT_ASSERT_ARG(idx);
if (size < sizeof(struct git_midx_header) + GIT_OID_RAWSZ)
return midx_error("multi-pack index is too short");
hdr = ((struct git_midx_header *)data);
if (hdr->signature != htonl(MIDX_SIGNATURE) ||
hdr->version != MIDX_VERSION ||
hdr->object_id_version != MIDX_OBJECT_ID_VERSION) {
return midx_error("unsupported multi-pack index version");
}
if (hdr->chunks == 0)
return midx_error("no chunks in multi-pack index");
/*
* The very first chunk's offset should be after the header, all the chunk
* headers, and a special zero chunk.
*/
last_chunk_offset =
sizeof(struct git_midx_header) +
(1 + hdr->chunks) * 12;
trailer_offset = size - GIT_OID_RAWSZ;
if (trailer_offset < last_chunk_offset)
return midx_error("wrong index size");
git_oid_cpy(&idx->checksum, (git_oid *)(data + trailer_offset));
if (git_hash_buf(&idx_checksum, data, (size_t)trailer_offset) < 0)
return midx_error("could not calculate signature");
if (!git_oid_equal(&idx_checksum, &idx->checksum))
return midx_error("index signature mismatch");
chunk_hdr = data + sizeof(struct git_midx_header);
last_chunk = NULL;
for (i = 0; i < hdr->chunks; ++i, chunk_hdr += 12) {
chunk_offset = ((off64_t)ntohl(*((uint32_t *)(chunk_hdr + 4)))) << 32 |
((off64_t)ntohl(*((uint32_t *)(chunk_hdr + 8))));
if (chunk_offset < last_chunk_offset)
return midx_error("chunks are non-monotonic");
if (chunk_offset >= trailer_offset)
return midx_error("chunks extend beyond the trailer");
if (last_chunk != NULL)
last_chunk->length = (size_t)(chunk_offset - last_chunk_offset);
last_chunk_offset = chunk_offset;
switch (ntohl(*((uint32_t *)(chunk_hdr + 0)))) {
case MIDX_PACKFILE_NAMES_ID:
chunk_packfile_names.offset = last_chunk_offset;
last_chunk = &chunk_packfile_names;
break;
case MIDX_OID_FANOUT_ID:
chunk_oid_fanout.offset = last_chunk_offset;
last_chunk = &chunk_oid_fanout;
break;
case MIDX_OID_LOOKUP_ID:
chunk_oid_lookup.offset = last_chunk_offset;
last_chunk = &chunk_oid_lookup;
break;
case MIDX_OBJECT_OFFSETS_ID:
chunk_object_offsets.offset = last_chunk_offset;
last_chunk = &chunk_object_offsets;
break;
case MIDX_OBJECT_LARGE_OFFSETS_ID:
chunk_object_large_offsets.offset = last_chunk_offset;
last_chunk = &chunk_object_large_offsets;
break;
default:
return midx_error("unrecognized chunk ID");
}
}
last_chunk->length = (size_t)(trailer_offset - last_chunk_offset);
error = midx_parse_packfile_names(
idx, data, ntohl(hdr->packfiles), &chunk_packfile_names);
if (error < 0)
return error;
error = midx_parse_oid_fanout(idx, data, &chunk_oid_fanout);
if (error < 0)
return error;
error = midx_parse_oid_lookup(idx, data, &chunk_oid_lookup);
if (error < 0)
return error;
error = midx_parse_object_offsets(idx, data, &chunk_object_offsets);
if (error < 0)
return error;
error = midx_parse_object_large_offsets(idx, data, &chunk_object_large_offsets);
if (error < 0)
return error;
return 0;
}
int git_midx_open(
git_midx_file **idx_out,
const char *path)
{
git_midx_file *idx;
git_file fd = -1;
size_t idx_size;
struct stat st;
int error;
/* TODO: properly open the file without access time using O_NOATIME */
fd = git_futils_open_ro(path);
if (fd < 0)
return fd;
if (p_fstat(fd, &st) < 0) {
p_close(fd);
git_error_set(GIT_ERROR_ODB, "multi-pack-index file not found - '%s'", path);
return -1;
}
if (!S_ISREG(st.st_mode) || !git__is_sizet(st.st_size)) {
p_close(fd);
git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path);
return -1;
}
idx_size = (size_t)st.st_size;
idx = git__calloc(1, sizeof(git_midx_file));
GIT_ERROR_CHECK_ALLOC(idx);
error = git_buf_sets(&idx->filename, path);
if (error < 0)
return error;
error = git_futils_mmap_ro(&idx->index_map, fd, 0, idx_size);
p_close(fd);
if (error < 0) {
git_midx_free(idx);
return error;
}
if ((error = git_midx_parse(idx, idx->index_map.data, idx_size)) < 0) {
git_midx_free(idx);
return error;
}
*idx_out = idx;
return 0;
}
bool git_midx_needs_refresh(
const git_midx_file *idx,
const char *path)
{
git_file fd = -1;
struct stat st;
ssize_t bytes_read;
git_oid idx_checksum = {{0}};
/* TODO: properly open the file without access time using O_NOATIME */
fd = git_futils_open_ro(path);
if (fd < 0)
return true;
if (p_fstat(fd, &st) < 0) {
p_close(fd);
return true;
}
if (!S_ISREG(st.st_mode) ||
!git__is_sizet(st.st_size) ||
(size_t)st.st_size != idx->index_map.len) {
p_close(fd);
return true;
}
if (p_lseek(fd, -GIT_OID_RAWSZ, SEEK_END) < 0) {
p_close(fd);
return true;
}
bytes_read = p_read(fd, &idx_checksum, GIT_OID_RAWSZ);
p_close(fd);
if (bytes_read != GIT_OID_RAWSZ)
return true;
return !git_oid_equal(&idx_checksum, &idx->checksum);
}
int git_midx_entry_find(
git_midx_entry *e,
git_midx_file *idx,
const git_oid *short_oid,
size_t len)
{
int pos, found = 0;
size_t pack_index;
uint32_t hi, lo;
const git_oid *current = NULL;
const unsigned char *object_offset;
off64_t offset;
GIT_ASSERT_ARG(idx);
hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]);
lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1]));
pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_RAWSZ, lo, hi, short_oid->id);
if (pos >= 0) {
/* An object matching exactly the oid was found */
found = 1;
current = idx->oid_lookup + pos;
} else {
/* No object was found */
/* pos refers to the object with the "closest" oid to short_oid */
pos = -1 - pos;
if (pos < (int)idx->num_objects) {
current = idx->oid_lookup + pos;
if (!git_oid_ncmp(short_oid, current, len))
found = 1;
}
}
if (found && len != GIT_OID_HEXSZ && pos + 1 < (int)idx->num_objects) {
/* Check for ambiguousity */
const git_oid *next = current + 1;
if (!git_oid_ncmp(short_oid, next, len)) {
found = 2;
}
}
if (!found)
return git_odb__error_notfound("failed to find offset for multi-pack index entry", short_oid, len);
if (found > 1)
return git_odb__error_ambiguous("found multiple offsets for multi-pack index entry");
object_offset = idx->object_offsets + pos * 8;
offset = ntohl(*((uint32_t *)(object_offset + 4)));
if (offset & 0x80000000) {
uint32_t object_large_offsets_pos = offset & 0x7fffffff;
const unsigned char *object_large_offsets_index = idx->object_large_offsets;
/* Make sure we're not being sent out of bounds */
if (object_large_offsets_pos >= idx->num_object_large_offsets)
return git_odb__error_notfound("invalid index into the object large offsets table", short_oid, len);
object_large_offsets_index += 8 * object_large_offsets_pos;
offset = (((uint64_t)ntohl(*((uint32_t *)(object_large_offsets_index + 0)))) << 32) |
ntohl(*((uint32_t *)(object_large_offsets_index + 4)));
}
pack_index = ntohl(*((uint32_t *)(object_offset + 0)));
if (pack_index >= git_vector_length(&idx->packfile_names))
return midx_error("invalid index into the packfile names table");
e->pack_index = pack_index;
e->offset = offset;
git_oid_cpy(&e->sha1, current);
return 0;
}
int git_midx_foreach_entry(
git_midx_file *idx,
git_odb_foreach_cb cb,
void *data)
{
size_t i;
int error;
GIT_ASSERT_ARG(idx);
for (i = 0; i < idx->num_objects; ++i) {
if ((error = cb(&idx->oid_lookup[i], data)) != 0)
return git_error_set_after_callback(error);
}
return error;
}
int git_midx_close(git_midx_file *idx)
{
GIT_ASSERT_ARG(idx);
if (idx->index_map.data)
git_futils_mmap_free(&idx->index_map);
git_vector_free(&idx->packfile_names);
return 0;
}
void git_midx_free(git_midx_file *idx)
{
if (!idx)
return;
git_buf_dispose(&idx->filename);
git_midx_close(idx);
git__free(idx);
}