blob: add git_blob_create_fromchunks()
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
diff --git a/include/git2/blob.h b/include/git2/blob.h
index 5517706..544dc7c 100644
--- a/include/git2/blob.h
+++ b/include/git2/blob.h
@@ -115,6 +115,48 @@ GIT_EXTERN(int) git_blob_create_fromfile(git_oid *oid, git_repository *repo, con
*/
GIT_EXTERN(int) git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path);
+/**
+ * Write a loose blob to the Object Database from a
+ * provider of chunks of data.
+ *
+ * Provided the `hintpath` parameter is filled, its value
+ * will help to determine what git filters should be applied
+ * to the object before it can be placed to the object database.
+ *
+ *
+ * The implementation of the callback has to respect the
+ * following rules:
+ *
+ * - `content` will have to be filled by the consumer. The maximum number
+ * of bytes that the buffer can accept per call is defined by the
+ * `max_length` parameter. Allocation and freeing of the buffer will be taken
+ * care of by the function.
+ *
+ * - The callback is expected to return the number of bytes
+ * that `content` have been filled with.
+ *
+ * - When there is no more data to stream, the callback should
+ * return 0. This will prevent it from being invoked anymore.
+ *
+ * - When an error occurs, the callback should return -1.
+ *
+ *
+ * @param oid Return the id of the written blob
+ *
+ * @param repo repository where the blob will be written.
+ * This repository can be bare or not.
+ *
+ * @param hintpath if not NULL, will help selecting the filters
+ * to apply onto the content of the blob to be created.
+ *
+ * @return GIT_SUCCESS or an error code
+ */
+GIT_EXTERN(int) git_blob_create_fromchunks(
+ git_oid *oid,
+ git_repository *repo,
+ const char *hintpath,
+ int (*source_cb)(char *content, size_t max_length, void *payload),
+ void *payload);
/**
* Write an in-memory buffer to the ODB as a blob
diff --git a/src/blob.c b/src/blob.c
index e25944b..699adec 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -148,27 +148,31 @@ static int write_symlink(
return error;
}
-static int blob_create_internal(git_oid *oid, git_repository *repo, const char *path)
+static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters)
{
int error;
struct stat st;
git_odb *odb = NULL;
git_off_t size;
- if ((error = git_path_lstat(path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0)
+ assert(hint_path || !try_load_filters);
+
+ if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0)
return error;
size = st.st_size;
if (S_ISLNK(st.st_mode)) {
- error = write_symlink(oid, odb, path, (size_t)size);
+ error = write_symlink(oid, odb, content_path, (size_t)size);
} else {
git_vector write_filters = GIT_VECTOR_INIT;
- int filter_count;
+ int filter_count = 0;
- /* Load the filters for writing this file to the ODB */
- filter_count = git_filters_load(
- &write_filters, repo, path, GIT_FILTER_TO_ODB);
+ if (try_load_filters) {
+ /* Load the filters for writing this file to the ODB */
+ filter_count = git_filters_load(
+ &write_filters, repo, hint_path, GIT_FILTER_TO_ODB);
+ }
if (filter_count < 0) {
/* Negative value means there was a critical error */
@@ -176,10 +180,10 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char *
} else if (filter_count == 0) {
/* No filters need to be applied to the document: we can stream
* directly from disk */
- error = write_file_stream(oid, odb, path, size);
+ error = write_file_stream(oid, odb, content_path, size);
} else {
/* We need to apply one or more filters */
- error = write_file_filtered(oid, odb, path, &write_filters);
+ error = write_file_filtered(oid, odb, content_path, &write_filters);
}
git_filters_free(&write_filters);
@@ -216,7 +220,7 @@ int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *pat
return -1;
}
- error = blob_create_internal(oid, repo, git_buf_cstr(&full_path));
+ error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
git_buf_free(&full_path);
return error;
@@ -232,8 +236,53 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat
return error;
}
- error = blob_create_internal(oid, repo, git_buf_cstr(&full_path));
+ error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
git_buf_free(&full_path);
return error;
}
+
+#define BUFFER_SIZE 4096
+
+int git_blob_create_fromchunks(
+ git_oid *oid,
+ git_repository *repo,
+ const char *hintpath,
+ int (*source_cb)(char *content, size_t max_length, void *payload),
+ void *payload)
+{
+ int error = -1, read_bytes;
+ char *content = NULL;
+ git_filebuf file = GIT_FILEBUF_INIT;
+
+ content = git__malloc(BUFFER_SIZE);
+ GITERR_CHECK_ALLOC(content);
+
+ if (git_filebuf_open(&file, hintpath == NULL ? "streamed" : hintpath, GIT_FILEBUF_TEMPORARY) < 0)
+ goto cleanup;
+
+ while (1) {
+ read_bytes = source_cb(content, BUFFER_SIZE, payload);
+
+ assert(read_bytes <= BUFFER_SIZE);
+
+ if (read_bytes <= 0)
+ break;
+
+ if (git_filebuf_write(&file, content, read_bytes) < 0)
+ goto cleanup;
+ }
+
+ if (read_bytes < 0)
+ goto cleanup;
+
+ if (git_filebuf_flush(&file) < 0)
+ goto cleanup;
+
+ error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL);
+
+cleanup:
+ git_filebuf_cleanup(&file);
+ git__free(content);
+ return error;
+}
diff --git a/tests-clar/object/blob/fromchunks.c b/tests-clar/object/blob/fromchunks.c
new file mode 100644
index 0000000..228e969
--- /dev/null
+++ b/tests-clar/object/blob/fromchunks.c
@@ -0,0 +1,87 @@
+#include "clar_libgit2.h"
+#include "buffer.h"
+#include "posix.h"
+#include "path.h"
+#include "fileops.h"
+
+static git_repository *repo;
+static char textual_content[] = "libgit2\n\r\n\0";
+
+void test_object_blob_fromchunks__initialize(void)
+{
+ repo = cl_git_sandbox_init("testrepo.git");
+}
+
+void test_object_blob_fromchunks__cleanup(void)
+{
+ cl_git_sandbox_cleanup();
+}
+
+static int text_chunked_source_cb(char *content, size_t max_length, void *payload)
+{
+ int *count;
+
+ GIT_UNUSED(max_length);
+
+ count = (int *)payload;
+ (*count)--;
+
+ if (*count == 0)
+ return 0;
+
+ strcpy(content, textual_content);
+ return strlen(textual_content);
+}
+
+void test_object_blob_fromchunks__can_create_a_blob_from_a_in_memory_chunk_provider(void)
+{
+ git_oid expected_oid, oid;
+ git_object *blob;
+ int howmany = 7;
+
+ cl_git_pass(git_oid_fromstr(&expected_oid, "321cbdf08803c744082332332838df6bd160f8f9"));
+
+ cl_git_fail(git_object_lookup(&blob, repo, &expected_oid, GIT_OBJ_ANY));
+
+ cl_git_pass(git_blob_create_fromchunks(&oid, repo, NULL, text_chunked_source_cb, &howmany));
+
+ cl_git_pass(git_object_lookup(&blob, repo, &expected_oid, GIT_OBJ_ANY));
+ git_object_free(blob);
+}
+
+#define GITATTR "* text=auto\n" \
+ "*.txt text\n" \
+ "*.data binary\n"
+
+static void write_attributes(git_repository *repo)
+{
+ git_buf buf = GIT_BUF_INIT;
+
+ cl_git_pass(git_buf_joinpath(&buf, git_repository_path(repo), "info"));
+ cl_git_pass(git_buf_joinpath(&buf, git_buf_cstr(&buf), "attributes"));
+
+ cl_git_pass(git_futils_mkpath2file(git_buf_cstr(&buf), 0777));
+ cl_git_rewritefile(git_buf_cstr(&buf), GITATTR);
+
+ git_buf_free(&buf);
+}
+
+static void assert_named_chunked_blob(const char *expected_sha, const char *fake_name)
+{
+ git_oid expected_oid, oid;
+ int howmany = 7;
+
+ cl_git_pass(git_oid_fromstr(&expected_oid, expected_sha));
+
+ cl_git_pass(git_blob_create_fromchunks(&oid, repo, fake_name, text_chunked_source_cb, &howmany));
+ cl_assert(git_oid_cmp(&expected_oid, &oid) == 0);
+}
+
+void test_object_blob_fromchunks__creating_a_blob_from_chunks_honors_the_attributes_directives(void)
+{
+ write_attributes(repo);
+
+ assert_named_chunked_blob("321cbdf08803c744082332332838df6bd160f8f9", "dummy.data");
+ assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.txt");
+ assert_named_chunked_blob("e9671e138a780833cb689753570fd10a55be84fb", "dummy.dunno");
+}