Commit 6cd094ef6c54ce24b44fb2d392b37ca23bc512a2

Edward Thomson 2020-06-04T07:40:35

Merge pull request #5540 from libgit2/ethomson/1_0_1 Release v1.0.1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
diff --git a/deps/ntlmclient/compat.h b/deps/ntlmclient/compat.h
index 555fa3f..f4d859a 100644
--- a/deps/ntlmclient/compat.h
+++ b/deps/ntlmclient/compat.h
@@ -25,7 +25,7 @@
 /* See man page endian(3) */
 # include <endian.h>
 # define htonll htobe64
-#elif defined(__OpenBSD__)
+#elif defined(__NetBSD__) || defined(__OpenBSD__)
 /* See man page htobe64(3) */
 # include <endian.h>
 # define htonll htobe64
diff --git a/docs/changelog.md b/docs/changelog.md
index 97cf94c..c2423c3 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -1,3 +1,33 @@
+v1.0.1
+------
+
+This is a bugfix release with the following changes:
+
+- Calculating information about renamed files during merges is more
+  efficient because dissimilarity about files is now being cached and
+  no longer needs to be recomputed.
+  
+- The `git_worktree_prune_init_options` has been correctly restored for
+  backward compatibility.  In v1.0 it was incorrectly deprecated with a
+  typo.
+
+- The optional ntlmclient dependency now supports NetBSD.
+
+- A bug where attempting to stash on a bare repository may have failed
+  has been fixed.
+
+- Configuration files that are unreadable due to permissions are now
+  silently ignored, and treated as if they do not exist.  This matches
+  git's behavior; previously this case would have been an error.
+
+- v4 index files are now correctly written; previously we would read
+  them correctly but would not write the prefix-compression accurately,
+  causing corruption.
+
+- A bug where the smart HTTP transport could not read large data packets
+  has been fixed.  Previously, fetching from servers like Gerrit, that
+  sent large data packets, would error.
+
 v1.0
 ----
 
diff --git a/script/release.py b/script/release.py
index e0f2953..3d8e9b8 100755
--- a/script/release.py
+++ b/script/release.py
@@ -56,6 +56,17 @@ def verify_version(version):
         if v[0] != v[1]:
             raise Error("version.h: define '{}' does not match (got '{}', expected '{}')".format(k, v[0], v[1]))
 
+    with open('package.json') as f:
+        pkg = json.load(f)
+
+    try:
+        pkg_version = Version(pkg["version"])
+    except KeyError as err:
+        raise Error("package.json: missing the field {}".format(err))
+
+    if pkg_version != version:
+        raise Error("package.json: version does not match (got '{}', expected '{}')".format(pkg_version, version))
+
 def generate_relnotes(tree, version):
     with open('docs/changelog.md') as f:
         lines = f.readlines()
diff --git a/src/config_file.c b/src/config_file.c
index c9e3649..b1e0028 100644
--- a/src/config_file.c
+++ b/src/config_file.c
@@ -111,6 +111,15 @@ static int config_file_open(git_config_backend *cfg, git_config_level_t level, c
 	if (!git_path_exists(b->file.path))
 		return 0;
 
+	/*
+	 * git silently ignores configuration files that are not
+	 * readable.  We emulate that behavior.  This is particularly
+	 * important for sandboxed applications on macOS where the
+	 * git configuration files may not be readable.
+	 */
+	if (p_access(b->file.path, R_OK) < 0)
+		return GIT_ENOTFOUND;
+
 	if (res < 0 || (res = config_file_read(b->entries, repo, &b->file, level, 0)) < 0) {
 		git_config_entries_free(b->entries);
 		b->entries = NULL;
diff --git a/src/index.c b/src/index.c
index 907bd6d..36a8bdb 100644
--- a/src/index.c
+++ b/src/index.c
@@ -2744,7 +2744,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry, const cha
 			++same_len;
 		}
 		path_len -= same_len;
-		varint_len = git_encode_varint(NULL, 0, same_len);
+		varint_len = git_encode_varint(NULL, 0, strlen(last) - same_len);
 	}
 
 	disk_size = index_entry_size(path_len, varint_len, entry->flags);
@@ -2795,7 +2795,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry, const cha
 
 	if (last) {
 		varint_len = git_encode_varint((unsigned char *) path,
-					  disk_size, same_len);
+					  disk_size, strlen(last) - same_len);
 		assert(varint_len > 0);
 		path += varint_len;
 		disk_size -= varint_len;
diff --git a/src/merge.c b/src/merge.c
index 05a776e..afe69e5 100644
--- a/src/merge.c
+++ b/src/merge.c
@@ -68,6 +68,16 @@ struct merge_diff_df_data {
 	git_merge_diff *prev_conflict;
 };
 
+/*
+ * This acts as a negative cache entry marker. In case we've tried to calculate
+ * similarity metrics for a given blob already but `git_hashsig` determined
+ * that it's too small in order to have a meaningful hash signature, we will
+ * insert the address of this marker instead of `NULL`. Like this, we can
+ * easily check whether we have checked a gien entry already and skip doing the
+ * calculation again and again.
+ */
+static int cache_invalid_marker;
+
 /* Merge base computation */
 
 int merge_bases_many(git_commit_list **out, git_revwalk **walk_out, git_repository *repo, size_t length, const git_oid input_array[])
@@ -1027,6 +1037,9 @@ static int index_entry_similarity_calc(
 	git_object_size_t blobsize;
 	int error;
 
+	if (*out || *out == &cache_invalid_marker)
+		return 0;
+
 	*out = NULL;
 
 	if ((error = git_blob_lookup(&blob, repo, &entry->id)) < 0)
@@ -1047,6 +1060,8 @@ static int index_entry_similarity_calc(
 	error = opts->metric->buffer_signature(out, &diff_file,
 		git_blob_rawcontent(blob), (size_t)blobsize,
 		opts->metric->payload);
+	if (error == GIT_EBUFS)
+		*out = &cache_invalid_marker;
 
 	git_blob_free(blob);
 
@@ -1069,18 +1084,16 @@ static int index_entry_similarity_inexact(
 		return 0;
 
 	/* update signature cache if needed */
-	if (!cache[a_idx] && (error = index_entry_similarity_calc(&cache[a_idx], repo, a, opts)) < 0)
-		return error;
-	if (!cache[b_idx] && (error = index_entry_similarity_calc(&cache[b_idx], repo, b, opts)) < 0)
+	if ((error = index_entry_similarity_calc(&cache[a_idx], repo, a, opts)) < 0 ||
+	    (error = index_entry_similarity_calc(&cache[b_idx], repo, b, opts)) < 0)
 		return error;
 
 	/* some metrics may not wish to process this file (too big / too small) */
-	if (!cache[a_idx] || !cache[b_idx])
+	if (cache[a_idx] == &cache_invalid_marker || cache[b_idx] == &cache_invalid_marker)
 		return 0;
 
 	/* compare signatures */
-	if (opts->metric->similarity(
-		&score, cache[a_idx], cache[b_idx], opts->metric->payload) < 0)
+	if (opts->metric->similarity(&score, cache[a_idx], cache[b_idx], opts->metric->payload) < 0)
 		return -1;
 
 	/* clip score */
@@ -1550,7 +1563,7 @@ int git_merge_diff_list__find_renames(
 done:
 	if (cache != NULL) {
 		for (i = 0; i < cache_size; ++i) {
-			if (cache[i] != NULL)
+			if (cache[i] != NULL && cache[i] != &cache_invalid_marker)
 				opts->metric->free_signature(cache[i], opts->metric->payload);
 		}
 
diff --git a/src/stash.c b/src/stash.c
index 4a13d05..790f56f 100644
--- a/src/stash.c
+++ b/src/stash.c
@@ -173,7 +173,7 @@ static int stash_to_index(
 	git_index *index,
 	const char *path)
 {
-	git_index *repo_index;
+	git_index *repo_index = NULL;
 	git_index_entry entry = {{0}};
 	struct stat st;
 	int error;
@@ -187,7 +187,7 @@ static int stash_to_index(
 		return error;
 
 	git_index_entry__init_from_stat(&entry, &st,
-		(repo_index != NULL || !repo_index->distrust_filemode));
+		(repo_index == NULL || !repo_index->distrust_filemode));
 
 	entry.path = path;
 
diff --git a/src/transports/httpclient.c b/src/transports/httpclient.c
index bde67ca..010baa6 100644
--- a/src/transports/httpclient.c
+++ b/src/transports/httpclient.c
@@ -1038,6 +1038,7 @@ on_error:
 
 GIT_INLINE(int) client_read(git_http_client *client)
 {
+	http_parser_context *parser_context = client->parser.data;
 	git_stream *stream;
 	char *buf = client->read_buf.ptr + client->read_buf.size;
 	size_t max_len;
@@ -1054,6 +1055,9 @@ GIT_INLINE(int) client_read(git_http_client *client)
 	max_len = client->read_buf.asize - client->read_buf.size;
 	max_len = min(max_len, INT_MAX);
 
+	if (parser_context->output_size)
+		max_len = min(max_len, parser_context->output_size);
+
 	if (max_len == 0) {
 		git_error_set(GIT_ERROR_HTTP, "no room in output buffer");
 		return -1;
@@ -1191,7 +1195,7 @@ static void complete_response_body(git_http_client *client)
 	/* If we're not keeping alive, don't bother. */
 	if (!client->keepalive) {
 		client->connected = 0;
-		return;
+		goto done;
 	}
 
 	parser_context.client = client;
@@ -1205,6 +1209,9 @@ static void complete_response_body(git_http_client *client)
 		git_error_clear();
 		client->connected = 0;
 	}
+
+done:
+	git_buf_clear(&client->read_buf);
 }
 
 int git_http_client_send_request(
@@ -1419,15 +1426,20 @@ int git_http_client_read_body(
 	client->parser.data = &parser_context;
 
 	/*
-	 * Clients expect to get a non-zero amount of data from us.
-	 * With a sufficiently small buffer, one might only read a chunk
-	 * length.  Loop until we actually have data to return.
+	 * Clients expect to get a non-zero amount of data from us,
+	 * so we either block until we have data to return, until we
+	 * hit EOF or there's an error.  Do this in a loop, since we
+	 * may end up reading only some stream metadata (like chunk
+	 * information).
 	 */
 	while (!parser_context.output_written) {
 		error = client_read_and_parse(client);
 
 		if (error <= 0)
 			goto done;
+
+		if (client->state == DONE)
+			break;
 	}
 
 	assert(parser_context.output_written <= INT_MAX);
diff --git a/src/worktree.c b/src/worktree.c
index e171afb..74b32f9 100644
--- a/src/worktree.c
+++ b/src/worktree.c
@@ -506,7 +506,7 @@ int git_worktree_prune_options_init(
 	return 0;
 }
 
-int git_worktree_pruneinit_options(git_worktree_prune_options *opts,
+int git_worktree_prune_init_options(git_worktree_prune_options *opts,
 	unsigned int version)
 {
 	return git_worktree_prune_options_init(opts, version);
diff --git a/tests/config/read.c b/tests/config/read.c
index 008dfd9..ba97302 100644
--- a/tests/config/read.c
+++ b/tests/config/read.c
@@ -849,6 +849,23 @@ void test_config_read__invalid_quoted_third_section(void)
 	git_config_free(cfg);
 }
 
+void test_config_read__unreadable_file_ignored(void)
+{
+	git_buf buf = GIT_BUF_INIT;
+	git_config *cfg;
+	int ret;
+
+	cl_set_cleanup(&clean_test_config, NULL);
+	cl_git_mkfile("./testconfig", "[some] var = value\n[some \"OtheR\"] var = value");
+	cl_git_pass(p_chmod("./testconfig", 0));
+
+	ret = git_config_open_ondisk(&cfg, "./test/config");
+	cl_assert(ret == 0 || ret == GIT_ENOTFOUND);
+
+	git_config_free(cfg);
+	git_buf_dispose(&buf);
+}
+
 void test_config_read__single_line(void)
 {
 	git_buf buf = GIT_BUF_INIT;
diff --git a/tests/index/version.c b/tests/index/version.c
index 3827df8..b6c0b79 100644
--- a/tests/index/version.c
+++ b/tests/index/version.c
@@ -43,6 +43,7 @@ void test_index_version__can_write_v4(void)
 	    "xz",
 	    "xyzzyx"
 	};
+	git_repository *repo;
 	git_index_entry entry;
 	git_index *index;
 	size_t i;
@@ -63,7 +64,8 @@ void test_index_version__can_write_v4(void)
 	cl_git_pass(git_index_write(index));
 	git_index_free(index);
 
-	cl_git_pass(git_repository_index(&index, g_repo));
+	cl_git_pass(git_repository_open(&repo, git_repository_path(g_repo)));
+	cl_git_pass(git_repository_index(&index, repo));
 	cl_assert(git_index_version(index) == 4);
 
 	for (i = 0; i < ARRAY_SIZE(paths); i++) {
@@ -74,6 +76,7 @@ void test_index_version__can_write_v4(void)
 	}
 
 	git_index_free(index);
+	git_repository_free(repo);
 }
 
 void test_index_version__v4_uses_path_compression(void)
diff --git a/tests/merge/trees/renames.c b/tests/merge/trees/renames.c
index e0b12af..c515aaf 100644
--- a/tests/merge/trees/renames.c
+++ b/tests/merge/trees/renames.c
@@ -274,3 +274,80 @@ void test_merge_trees_renames__submodules(void)
 	cl_assert(merge_test_index(index, merge_index_entries, 7));
 	git_index_free(index);
 }
+
+void test_merge_trees_renames__cache_recomputation(void)
+{
+	git_oid blob, binary, ancestor_oid, theirs_oid, ours_oid;
+	git_merge_options opts = GIT_MERGE_OPTIONS_INIT;
+	git_buf path = GIT_BUF_INIT;
+	git_treebuilder *builder;
+	git_tree *ancestor_tree, *their_tree, *our_tree;
+	git_index *index;
+	size_t blob_size;
+	void *data;
+	size_t i;
+
+	cl_git_pass(git_oid_fromstr(&blob, "a2d8d1824c68541cca94ffb90f79291eba495921"));
+
+	/*
+	 * Create a 50MB blob that consists of NUL bytes only. It is important
+	 * that this blob is of a special format, most importantly it cannot
+	 * contain more than four non-consecutive newlines or NUL bytes. This
+	 * is because of git_hashsig's inner workings where all files with less
+	 * than four "lines" are deemed to small.
+	 */
+	blob_size = 50 * 1024 * 1024;
+	cl_assert(data = git__calloc(blob_size, 1));
+	cl_git_pass(git_blob_create_from_buffer(&binary, repo, data, blob_size));
+
+	/*
+	 * Create the common ancestor, which has 1000 dummy blobs and the binary
+	 * blob. The dummy blobs serve as potential rename targets for the
+	 * dummy blob.
+	 */
+	cl_git_pass(git_treebuilder_new(&builder, repo, NULL));
+	for (i = 0; i < 1000; i++) {
+		cl_git_pass(git_buf_printf(&path, "%"PRIuMAX".txt", i));
+		cl_git_pass(git_treebuilder_insert(NULL, builder, path.ptr, &blob, GIT_FILEMODE_BLOB));
+		git_buf_clear(&path);
+	}
+	cl_git_pass(git_treebuilder_insert(NULL, builder, "original.bin", &binary, GIT_FILEMODE_BLOB));
+	cl_git_pass(git_treebuilder_write(&ancestor_oid, builder));
+
+	/* We now the binary blob in our tree. */
+	cl_git_pass(git_treebuilder_remove(builder, "original.bin"));
+	cl_git_pass(git_treebuilder_insert(NULL, builder, "renamed.bin", &binary, GIT_FILEMODE_BLOB));
+	cl_git_pass(git_treebuilder_write(&ours_oid, builder));
+
+	git_treebuilder_free(builder);
+
+	/* And move everything into a subdirectory in their tree. */
+	cl_git_pass(git_treebuilder_new(&builder, repo, NULL));
+	cl_git_pass(git_treebuilder_insert(NULL, builder, "subdir", &ancestor_oid, GIT_FILEMODE_TREE));
+	cl_git_pass(git_treebuilder_write(&theirs_oid, builder));
+
+	/*
+	 * Now merge ancestor, ours and theirs. As `git_hashsig` refuses to
+	 * create a hash signature for the 50MB binary file, we historically
+	 * didn't cache the hashsig computation for it. As a result, we now
+	 * started looking up the 50MB blob and scanning it at least 1000
+	 * times, which takes a long time.
+	 *
+	 * The number of 1000 blobs is chosen in such a way that it's
+	 * noticeable when the bug creeps in again, as it takes around 12
+	 * minutes on my machine to compute the following merge.
+	 */
+	opts.target_limit = 5000;
+	cl_git_pass(git_tree_lookup(&ancestor_tree, repo, &ancestor_oid));
+	cl_git_pass(git_tree_lookup(&their_tree, repo, &theirs_oid));
+	cl_git_pass(git_tree_lookup(&our_tree, repo, &ours_oid));
+	cl_git_pass(git_merge_trees(&index, repo, ancestor_tree, our_tree, their_tree, &opts));
+
+	git_treebuilder_free(builder);
+	git_buf_dispose(&path);
+	git_index_free(index);
+	git_tree_free(ancestor_tree);
+	git_tree_free(their_tree);
+	git_tree_free(our_tree);
+	git__free(data);
+}
diff --git a/tests/online/clone.c b/tests/online/clone.c
index 034d0c2..9107956 100644
--- a/tests/online/clone.c
+++ b/tests/online/clone.c
@@ -11,6 +11,7 @@
 #define BB_REPO_URL "https://libgit3@bitbucket.org/libgit2/testgitrepository.git"
 #define BB_REPO_URL_WITH_PASS "https://libgit3:libgit3@bitbucket.org/libgit2/testgitrepository.git"
 #define BB_REPO_URL_WITH_WRONG_PASS "https://libgit3:wrong@bitbucket.org/libgit2/testgitrepository.git"
+#define GOOGLESOURCE_REPO_URL "https://chromium.googlesource.com/external/github.com/sergi/go-diff"
 
 #define SSH_REPO_URL "ssh://github.com/libgit2/TestGitRepository"
 
@@ -463,6 +464,13 @@ void test_online_clone__bitbucket_falls_back_to_specified_creds(void)
 	cl_fixture_cleanup("./foo");
 }
 
+void test_online_clone__googlesource(void)
+{
+	cl_git_pass(git_clone(&g_repo, GOOGLESOURCE_REPO_URL, "./foo", &g_options));
+	git_repository_free(g_repo); g_repo = NULL;
+	cl_fixture_cleanup("./foo");
+}
+
 static int cancel_at_half(const git_indexer_progress *stats, void *payload)
 {
 	GIT_UNUSED(payload);