blob: introduce creating a blob by writing into a stream The pair of `git_blob_create_frombuffer()` and `git_blob_create_frombuffer_commit()` is meant to replace `git_blob_create_fromchunks()` by providing a way for a user to write a new blob when they want filtering or they do not know the size. This approach allows the caller to retain control over when to add data to this buffer and a more natural fit into higher-level language's own stream abstractions instead of having to handle IO wait in the callback. The in-memory buffer size of 2MB is chosen somewhat arbitrarily to be a round multiple of usual page sizes and a value where most blobs seem likely to be either going to be way below or way over that size. It's also a round number of pages. This implementation re-uses the helper we have from `_fromchunks()` so we end up writing everything to disk, but hopefully more efficiently than with a default filebuf. A later optimisation can be to avoid writing the in-memory contents to disk, with some extra complexity.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
diff --git a/include/git2/blob.h b/include/git2/blob.h
index 9a57c37..f451593 100644
--- a/include/git2/blob.h
+++ b/include/git2/blob.h
@@ -192,6 +192,49 @@ GIT_EXTERN(int) git_blob_create_fromchunks(
void *payload);
/**
+ * Create a stream to write a new blob into the object db
+ *
+ * This function may need to buffer the data on disk and will in
+ * general not be the right choice if you know the size of the data
+ * to write. If you have data in memory, use
+ * `git_blob_create_frombuffer()`. If you do not, but know the size of
+ * the contents (and don't want/need to perform filtering), use
+ * `git_odb_open_wstream()`.
+ *
+ * Don't close this stream yourself but pass it to
+ * `git_blob_create_fromstream_commit()` to commit the write to the
+ * object db and get the object id.
+ *
+ * If the `hintpath` parameter is filled, it will be used to determine
+ * what git filters should be applied to the object before it is written
+ * to the object database.
+ *
+ * @param out the stream into which to write
+ * @param repo Repository where the blob will be written.
+ * This repository can be bare or not.
+ * @param hintpath If not NULL, will be used to select data filters
+ * to apply onto the content of the blob to be created.
+ * @return 0 or error code
+ */
+GIT_EXTERN(int) git_blob_create_fromstream(
+ git_writestream **out,
+ git_repository *repo,
+ const char *hintpath);
+
+/**
+ * Close the stream and write the blob to the object db
+ *
+ * The stream will be closed and freed.
+ *
+ * @param out the id of the new blob
+ * @param stream the stream to close
+ * @return 0 or an error code
+ */
+GIT_EXTERN(int) git_blob_create_fromstream_commit(
+ git_oid *out,
+ git_writestream *stream);
+
+/**
* Write an in-memory buffer to the ODB as a blob
*
* @param id return the id of the written blob
diff --git a/src/blob.c b/src/blob.c
index ad0f4ac..a1ef247 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -334,6 +334,98 @@ cleanup:
return error;
}
+typedef struct {
+ git_writestream parent;
+ git_filebuf fbuf;
+ git_repository *repo;
+ char *hintpath;
+} blob_writestream;
+
+static int blob_writestream_close(git_writestream *_stream)
+{
+ blob_writestream *stream = (blob_writestream *) _stream;
+
+ git_filebuf_cleanup(&stream->fbuf);
+ return 0;
+}
+
+static void blob_writestream_free(git_writestream *_stream)
+{
+ blob_writestream *stream = (blob_writestream *) _stream;
+
+ git_filebuf_cleanup(&stream->fbuf);
+ git__free(stream->hintpath);
+ git__free(stream);
+}
+
+static int blob_writestream_write(git_writestream *_stream, const char *buffer, size_t len)
+{
+ blob_writestream *stream = (blob_writestream *) _stream;
+
+ return git_filebuf_write(&stream->fbuf, buffer, len);
+}
+
+int git_blob_create_fromstream(git_writestream **out, git_repository *repo, const char *hintpath)
+{
+ int error;
+ git_buf path = GIT_BUF_INIT;
+ blob_writestream *stream;
+
+ assert(out && repo);
+
+ stream = git__calloc(1, sizeof(blob_writestream));
+ GITERR_CHECK_ALLOC(stream);
+
+ if (hintpath) {
+ stream->hintpath = git__strdup(hintpath);
+ GITERR_CHECK_ALLOC(stream->hintpath);
+ }
+
+ stream->repo = repo;
+ stream->parent.write = blob_writestream_write;
+ stream->parent.close = blob_writestream_close;
+ stream->parent.free = blob_writestream_free;
+
+ if ((error = git_buf_joinpath(&path,
+ git_repository_path(repo), GIT_OBJECTS_DIR "streamed")) < 0)
+ goto cleanup;
+
+ if ((error = git_filebuf_open_withsize(&stream->fbuf, git_buf_cstr(&path), GIT_FILEBUF_TEMPORARY,
+ 0666, 2 * 1024 * 1024)) < 0)
+ goto cleanup;
+
+ *out = (git_writestream *) stream;
+
+cleanup:
+ if (error < 0)
+ blob_writestream_free((git_writestream *) stream);
+
+ git_buf_free(&path);
+ return error;
+}
+
+int git_blob_create_fromstream_commit(git_oid *out, git_writestream *_stream)
+{
+ int error;
+ blob_writestream *stream = (blob_writestream *) _stream;
+
+ /*
+ * We can make this more officient by avoiding writing to
+ * disk, but for now let's re-use the helper functions we
+ * have.
+ */
+ if ((error = git_filebuf_flush(&stream->fbuf)) < 0)
+ goto cleanup;
+
+ error = git_blob__create_from_paths(out, NULL, stream->repo, stream->fbuf.path_lock,
+ stream->hintpath, 0, !!stream->hintpath);
+
+cleanup:
+ blob_writestream_free(_stream);
+ return error;
+
+}
+
int git_blob_is_binary(const git_blob *blob)
{
git_buf content = GIT_BUF_INIT;
diff --git a/tests/object/blob/fromstream.c b/tests/object/blob/fromstream.c
new file mode 100644
index 0000000..10f2d8b
--- /dev/null
+++ b/tests/object/blob/fromstream.c
@@ -0,0 +1,103 @@
+#include "clar_libgit2.h"
+#include "buffer.h"
+#include "posix.h"
+#include "path.h"
+#include "fileops.h"
+
+static git_repository *repo;
+static char textual_content[] = "libgit2\n\r\n\0";
+
+void test_object_blob_fromstream__initialize(void)
+{
+ repo = cl_git_sandbox_init("testrepo.git");
+}
+
+void test_object_blob_fromstream__cleanup(void)
+{
+ cl_git_sandbox_cleanup();
+}
+
+static int text_chunked_source_cb(char *content, size_t max_length, void *payload)
+{
+ int *count;
+
+ GIT_UNUSED(max_length);
+
+ count = (int *)payload;
+ (*count)--;
+
+ if (*count == 0)
+ return 0;
+
+ strcpy(content, textual_content);
+ return (int)strlen(textual_content);
+}
+
+void test_object_blob_fromstream__multiple_write(void)
+{
+ git_oid expected_id, id;
+ git_object *blob;
+ git_writestream *stream;
+ int i, howmany = 6;
+
+ cl_git_pass(git_oid_fromstr(&expected_id, "321cbdf08803c744082332332838df6bd160f8f9"));
+
+ cl_git_fail_with(GIT_ENOTFOUND,
+ git_object_lookup(&blob, repo, &expected_id, GIT_OBJ_ANY));
+
+ cl_git_pass(git_blob_create_fromstream(&stream, repo, NULL));
+
+ for (i = 0; i < howmany; i++)
+ cl_git_pass(stream->write(stream, textual_content, strlen(textual_content)));
+
+ cl_git_pass(git_blob_create_fromstream_end(&id, stream));
+ cl_assert_equal_oid(&expected_id, &id);
+
+ cl_git_pass(git_object_lookup(&blob, repo, &expected_id, GIT_OBJ_BLOB));
+
+ git_object_free(blob);
+}
+
+#define GITATTR "* text=auto\n" \
+ "*.txt text\n" \
+ "*.data binary\n"
+
+static void write_attributes(git_repository *repo)
+{
+ git_buf buf = GIT_BUF_INIT;
+
+ cl_git_pass(git_buf_joinpath(&buf, git_repository_path(repo), "info"));
+ cl_git_pass(git_buf_joinpath(&buf, git_buf_cstr(&buf), "attributes"));
+
+ cl_git_pass(git_futils_mkpath2file(git_buf_cstr(&buf), 0777));
+ cl_git_rewritefile(git_buf_cstr(&buf), GITATTR);
+
+ git_buf_free(&buf);
+}
+
+static void assert_named_chunked_blob(const char *expected_sha, const char *fake_name)
+{
+ git_oid expected_id, id;
+ git_writestream *stream;
+ int i, howmany = 6;
+
+ cl_git_pass(git_oid_fromstr(&expected_id, expected_sha));
+
+ cl_git_pass(git_blob_create_fromstream(&stream, repo, fake_name));
+
+ for (i = 0; i < howmany; i++)
+ cl_git_pass(stream->write(stream, textual_content, strlen(textual_content)));
+
+ cl_git_pass(git_blob_create_fromstream_end(&id, stream));
+
+ cl_assert_equal_oid(&expected_id, &id);
+}
+
+void test_object_blob_fromstream__creating_a_blob_from_chunks_honors_the_attributes_directives(void)
+{
+ write_attributes(repo);
+
+ assert_named_chunked_blob("321cbdf08803c744082332332838df6bd160f8f9", "dummy.data");
+ assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.txt");
+ assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.dunno");
+}