Skip to content

range-diff: add configurable memory limit for cost matrix #1958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

range-diff: add configurable memory limit for cost matrix #1958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions builtin/log.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1404,6 +1404,7 @@ static void make_cover_letter(struct rev_info *rev, int use_separate_file,
struct range_diff_options range_diff_opts = {
.creation_factor = rev->creation_factor,
.dual_color = 1,
.max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,
.diffopt = &opts,
.other_arg = &other_arg
};
Expand Down
30 changes: 26 additions & 4 deletions builtin/range-diff.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "parse-options.h"
#include "range-diff.h"
#include "config.h"
#include "parse.h"


static const char * const builtin_range_diff_usage[] = {
Expand All @@ -15,6 +16,22 @@ N_("git range-diff [<options>] <base> <old-tip> <new-tip>"),
NULL
};

static int parse_max_memory(const struct option *opt, const char *arg, int unset)
{
size_t *max_memory = opt->value;
uintmax_t val;

if (unset) {
return 0;
}

if (!git_parse_unsigned(arg, &val, SIZE_MAX))
return error(_("invalid max-memory value: %s"), arg);

*max_memory = (size_t)val;
return 0;
}

int cmd_range_diff(int argc,
const char **argv,
const char *prefix,
Expand All @@ -25,6 +42,7 @@ int cmd_range_diff(int argc,
struct strvec diff_merges_arg = STRVEC_INIT;
struct range_diff_options range_diff_opts = {
.creation_factor = RANGE_DIFF_CREATION_FACTOR_DEFAULT,
.max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,
.diffopt = &diffopt,
.other_arg = &other_arg
};
Expand All @@ -33,17 +51,21 @@ int cmd_range_diff(int argc,
OPT_INTEGER(0, "creation-factor",
&range_diff_opts.creation_factor,
N_("percentage by which creation is weighted")),
OPT_PASSTHRU_ARGV(0, "diff-merges", &diff_merges_arg,
N_("style"), N_("passed to 'git log'"), 0),
OPT_BOOL(0, "left-only", &left_only,
N_("only emit output related to the first range")),
OPT_CALLBACK(0, "max-memory", &range_diff_opts.max_memory,
N_("size"),
N_("maximum memory for cost matrix (default 4G)"),
parse_max_memory),
OPT_BOOL(0, "no-dual-color", &simple_color,
N_("use simple diff colors")),
OPT_PASSTHRU_ARGV(0, "notes", &other_arg,
N_("notes"), N_("passed to 'git log'"),
PARSE_OPT_OPTARG),
OPT_PASSTHRU_ARGV(0, "diff-merges", &diff_merges_arg,
N_("style"), N_("passed to 'git log'"), 0),
OPT_PASSTHRU_ARGV(0, "remerge-diff", &diff_merges_arg, NULL,
N_("passed to 'git log'"), PARSE_OPT_NOARG),
OPT_BOOL(0, "left-only", &left_only,
N_("only emit output related to the first range")),
OPT_BOOL(0, "right-only", &right_only,
N_("only emit output related to the second range")),
OPT_END()
Expand Down
1 change: 1 addition & 0 deletions log-tree.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ static void show_diff_of_diff(struct rev_info *opt)
struct range_diff_options range_diff_opts = {
.creation_factor = opt->creation_factor,
.dual_color = 1,
.max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,
.diffopt = &opts
};

Expand Down
35 changes: 25 additions & 10 deletions range-diff.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "apply.h"
#include "revision.h"


struct patch_util {
/* For the search for an exact match */
struct hashmap_entry e;
Expand Down Expand Up @@ -287,8 +288,8 @@ static void find_exact_matches(struct string_list *a, struct string_list *b)
}

static int diffsize_consume(void *data,
char *line UNUSED,
unsigned long len UNUSED)
char *line UNUSED,
unsigned long len UNUSED)
{
(*(int *)data)++;
return 0;
Expand Down Expand Up @@ -325,13 +326,24 @@ static int diffsize(const char *a, const char *b)
}

static void get_correspondences(struct string_list *a, struct string_list *b,
int creation_factor)
int creation_factor, size_t max_memory)
{
int n = a->nr + b->nr;
int *cost, c, *a2b, *b2a;
int i, j;

ALLOC_ARRAY(cost, st_mult(n, n));
size_t cost_size = st_mult(n, n);
size_t cost_bytes = st_mult(sizeof(int), cost_size);
if (cost_bytes >= max_memory) {
struct strbuf cost_str = STRBUF_INIT;
struct strbuf max_str = STRBUF_INIT;
strbuf_humanise_bytes(&cost_str, cost_bytes);
strbuf_humanise_bytes(&max_str, max_memory);
die(_("range-diff: unable to compute the range-diff, since it "
"exceeds the maximum memory for the cost matrix: %s "
"(%"PRIuMAX" bytes) needed, %s (%"PRIuMAX" bytes) available"),
cost_str.buf, (uintmax_t)cost_bytes, max_str.buf, (uintmax_t)max_memory);
}
ALLOC_ARRAY(cost, cost_size);
ALLOC_ARRAY(a2b, n);
ALLOC_ARRAY(b2a, n);

Expand All @@ -351,7 +363,8 @@ static void get_correspondences(struct string_list *a, struct string_list *b,
}

c = a_util->matching < 0 ?
a_util->diffsize * creation_factor / 100 : COST_MAX;
a_util->diffsize * creation_factor / 100 :
COST_MAX;
for (j = b->nr; j < n; j++)
cost[i + n * j] = c;
}
Expand All @@ -360,7 +373,8 @@ static void get_correspondences(struct string_list *a, struct string_list *b,
struct patch_util *util = b->items[j].util;

c = util->matching < 0 ?
util->diffsize * creation_factor / 100 : COST_MAX;
util->diffsize * creation_factor / 100 :
COST_MAX;
for (i = a->nr; i < n; i++)
cost[i + n * j] = c;
}
Expand Down Expand Up @@ -539,7 +553,7 @@ static void output(struct string_list *a, struct string_list *b,
if (i < a->nr && a_util->matching < 0) {
if (!range_diff_opts->right_only)
output_pair_header(&opts, patch_no_width,
&buf, &dashes, a_util, NULL);
&buf, &dashes, a_util, NULL);
i++;
continue;
}
Expand All @@ -548,7 +562,7 @@ static void output(struct string_list *a, struct string_list *b,
while (j < b->nr && b_util->matching < 0) {
if (!range_diff_opts->left_only)
output_pair_header(&opts, patch_no_width,
&buf, &dashes, NULL, b_util);
&buf, &dashes, NULL, b_util);
b_util = ++j < b->nr ? b->items[j].util : NULL;
}

Expand Down Expand Up @@ -591,7 +605,8 @@ int show_range_diff(const char *range1, const char *range2,
if (!res) {
find_exact_matches(&branch1, &branch2);
get_correspondences(&branch1, &branch2,
range_diff_opts->creation_factor);
range_diff_opts->creation_factor,
range_diff_opts->max_memory);
output(&branch1, &branch2, range_diff_opts);
}

Expand Down
5 changes: 5 additions & 0 deletions range-diff.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
#include "strvec.h"

#define RANGE_DIFF_CREATION_FACTOR_DEFAULT 60
#define RANGE_DIFF_MAX_MEMORY_DEFAULT \
(sizeof(void*) >= 8 ? \
((size_t)(1024L * 1024L) * (size_t)(4L * 1024L)) : /* 4GB on 64-bit */ \
((size_t)(1024L * 1024L) * (size_t)(2L * 1024L))) /* 2GB on 32-bit */

/*
* A much higher value than the default, when we KNOW we are comparing
Expand All @@ -17,6 +21,7 @@ struct range_diff_options {
unsigned dual_color:1;
unsigned left_only:1, right_only:1;
unsigned include_merges:1;
size_t max_memory;
const struct diff_options *diffopt; /* may be NULL */
const struct strvec *other_arg; /* may be NULL */
};
Expand Down
Loading