Move all scripts to bin/
[opensuse:expand-kernel-source.git] / bin / expand-kernel-source.pl
1 #!/usr/bin/perl
2
3 use strict;
4 use warnings;
5 no warnings 'recursion';
6
7 my $USAGE = "Usage: $0 --mainline <git> --suse <git> [--incremental] [[from]..[to:]]ref...\n";
8
9 BEGIN {
10         if ($0 =~ /^(.*)\/[^\/]*/) {
11                 push @INC, "$1/lib";
12                 $ENV{PATH} = "$ENV{PATH}:$1";
13         } else {
14                 push @INC,  "./lib";
15                 $ENV{PATH} = "$ENV{PATH}:.";
16         }
17 }
18
19 use Getopt::Long;
20 use SUSE::Kernel::Git;
21
22 my ($mainline_path, $suse_path, $force, $incremental, $expand_all, $jobs);
23 my ($mainline_git, $suse_git);
24
25 # ({ id, fullname, type }, ...)
26 my @refs;
27
28 # commit => 1
29 my %exclude_commits;
30
31 # suse rev => { author, tree, ... expanded => { tree, version, broken} }
32 my %revs;
33
34 # suse tree id => { tree => <expanded tree id>, broken => 0/1, version => ... }
35 my %suse_tree2expanded;
36 # expanded tree id => { same as above }
37 my %expanded_tree_data;
38
39 # suse commit => expanded commit
40 my $expanded_commits = GitTable->new("expanded-commits");
41
42 # mainline version => mainline commit
43 my $mainline_commits = GitTable->new("mainline-commits");
44
45 # full suse tree => patches subtree
46 my $subtrees = GitTable->new("subtrees");
47
48 # generic way of storing hashes in named blobs in git
49 package GitTable;
50
51 sub new {
52         my ($class, $name) = @_;
53
54         my $self = { _name => $name };
55         bless ($self, $class);
56         return $self;
57 }
58
59 sub load {
60         my $self = shift;
61
62         my $fd = $mainline_git->popen("cat-file", "blob", $self->{_name});
63         while (<$fd>) {
64                 chomp;
65                 next if /^#/;
66                 my ($key, $value) = split;
67                 if ($key =~ /^_/) {
68                         print STDERR "warning: invalid key $key in table $self->{_name}\n";
69                         next;
70                 }
71                 $self->{$key} = $value;
72         }
73         close($fd);
74 }
75
76 sub save {
77         my $self = shift;
78
79         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-w",
80                 "--stdin");
81         for my $key (sort(keys(%$self))) {
82                 next if $key =~ /^_/;
83                 printf $out "%s %s\n", $key, $self->{$key};
84         }
85         close($out);
86         my $hash = <$in>;
87         chomp $hash;
88         close($in);
89         waitpid($pid, 0);
90         die "Error saving $self->{_name} table\n" unless $? == 0 && $hash;
91         $mainline_git->read_cmd("tag", "-f", $self->{_name}, $hash);
92 }
93
94 package main;
95
96 sub load_suse_rev_data {
97         print STDERR "Loading commit data from $suse_path...\n";
98         my $total = scalar(keys(%revs));
99         my ($i, $percent_last) = (0, 0);
100         $subtrees->load();
101         for my $rev (keys(%revs)) {
102                 my $data = $suse_git->commit_data($rev);
103                 for my $key (keys(%$data)) {
104                         $revs{$rev}->{$key} = $data->{$key};
105                 }
106                 if ($suse_git->writable) {
107                         $revs{$rev}->{tree} = patches_subtree($revs{$rev}->{tree});
108                 }
109                 $suse_tree2expanded{$revs{$rev}->{tree}} ||= {};
110                 $revs{$rev}->{expanded} = $suse_tree2expanded{$revs{$rev}->{tree}};
111                 $i++;
112                 my $percent = int($i / $total * 100 + 0.5);
113                 if ($percent != $percent_last) {
114                         printf "%d%%\n", $percent;
115                 }
116                 $percent_last = $percent;
117         }
118         $subtrees->save();
119 }
120
121 # load suse-tree -> mainline-tree mapping from refs/heads/expanded-trees
122 sub load_expanded_trees {
123         my ($suse, $expanded, $version, $broken);
124
125         print STDERR "Loading expanded tree map from $mainline_path...\n";
126         my $fd = $mainline_git->popen("rev-list", "--no-merges",
127                 "--pretty=raw", "expanded-trees", "--");
128         while (<$fd>) {
129                 chomp;
130                 if (s/^tree //) {
131                         if ($suse) {
132                                 $suse_tree2expanded{$suse}{tree} = $expanded;
133                                 $suse_tree2expanded{$suse}{version} = $version;
134                                 $suse_tree2expanded{$suse}{broken} = $broken;
135                                 $expanded_tree_data{$expanded} =
136                                         $suse_tree2expanded{$suse};
137                         }
138                         $expanded = $_;
139                         ($suse, $version, $broken) = (undef, undef, undef);
140                 } elsif (s/^ *suse-tree: //) {
141                         $suse = $_;
142                 } elsif (s/^ *version: //) {
143                         $version = $_;
144                 } elsif (/^ *BROKEN/) {
145                         $broken = 1;
146                 }
147         }
148         close($fd);
149         if ($suse) {
150                 $suse_tree2expanded{$suse}{tree} = $expanded;
151                 $suse_tree2expanded{$suse}{version} = $version;
152                 $suse_tree2expanded{$suse}{broken} = $broken;
153                 $expanded_tree_data{$expanded} = $suse_tree2expanded{$suse};
154         }
155 }
156
157 # Replace, add or delete items in a given tree and return the new tree id
158 sub filter_tree {
159         my ($tree, $filter, $git) = @_;
160         $git ||= $suse_git;
161
162         my ($pid, $in, $out) = $git->popen2("mktree");
163         if ($tree) {
164                 my $ls_tree = $git->popen("ls-tree", $tree);
165                 while (<$ls_tree>) {
166                         chomp;
167                         my @old = split;
168                         my @new = &$filter(@old);
169                         next if !@new;
170                         # If the filter returns an empty list, the entry is deleted.
171                         # If the filter returns a list of four items (mode, type, hash
172                         # name), the item is either modified or left intact.
173                         # 8, 12, etc. returned items add entries to the tree.
174                         while (@new) {
175                                 printf $out "%s %s %s\t%s\n", splice(@new, 0, 4);
176                         }
177                 }
178                 close($ls_tree);
179         } else {
180                 my @new = &$filter();
181                 while (@new) {
182                         printf $out "%s %s %s\t%s\n", splice(@new, 0, 4);
183                 }
184         }
185         close($out);
186         my $res = <$in>;
187         chomp $res;
188         close($in);
189         waitpid($pid, 0);
190         die "Could not shrink tree $tree\n" unless $? == 0 && $res;
191         return $res;
192 }
193
194 # returns a tree that only contains series.conf, the patches.* directories
195 # and {scripts,rpm}/config.sh (to determine $SRCVERSION)
196 sub patches_subtree {
197         my $tree = shift;
198         my $res;
199
200         if (exists($subtrees->{$tree})) {
201                 return $subtrees->{$tree};
202         }
203         # first, filter scripts/ and rpm/
204         my $filter_config_sh = sub {
205                 return unless $_[3] eq "config.sh";
206                 return @_;
207         };
208         my $rpm_tree = filter_tree("$tree:rpm", $filter_config_sh);
209         my $scripts_tree = filter_tree("$tree:scripts", $filter_config_sh);
210         # now pick series.conf, patches.*/, replace scripts/ and rpm/
211         # and linux-*.tar.bz2 if present
212         my $filter = sub {
213                 my ($mode, $type, $hash, $name) = @_;
214
215                 if ($type eq "tree" && $name eq "rpm") {
216                         return ($mode, "tree", $rpm_tree, "rpm");
217                 } elsif ($type eq "tree" && $name eq "scripts") {
218                         return ($mode, "tree", $scripts_tree, "scripts");
219                 } elsif ($name =~ /^(series\.conf$|patches\.|linux-.*\.tar\.bz2$)/) {
220                         return ($mode, $type, $hash, $name);
221                 }
222                 return;
223         };
224         $res = filter_tree($tree, $filter);
225         $subtrees->{$tree} = $res;
226         return $res;
227 }
228
229 {
230 my $broken_file;
231 sub mark_tree_broken {
232         my $tree = shift;
233
234         if (!$broken_file) {
235                 my ($pid, $in, $out) = $mainline_git->popen2("hash-object",
236                         "-w", "--stdin");
237                 print $out "This patch series did not apply, a later commit will contain the changes.\n";
238                 close($out);
239                 $broken_file = <$in>;
240                 chomp $broken_file;
241                 close($in);
242                 waitpid($pid, 0);
243                 die "Error creating the BROKEN marker\n" unless $? == 0 &&
244                         $broken_file;
245         }
246         my $added;
247         my $filter = sub {
248                 my @entry = @_;
249                 my @res;
250                 my $cmp = 1;
251                 if (@entry) {
252                         $cmp = "BROKEN" cmp $entry[3];
253                 }
254                 if ($cmp == 0) {
255                         $added = 1;
256                 } elsif ($cmp == -1 && !$added) {
257                         # FIXME: breaks if there is no file while sorts after
258                         # "BROKEN"
259                         $added = 1;
260                         @res = ("100644", "blob", $broken_file, "BROKEN");
261                 }
262                 push(@res, @entry);
263                 return @res;
264         };
265         return filter_tree($tree, $filter, $mainline_git);
266 }
267 }
268
269 sub unmark_tree_broken {
270         my $tree = shift;
271
272         my $filter = sub {
273                 return if $_[3] eq "BROKEN";
274                 return @_;
275         };
276         return filter_tree($tree, $filter, $mainline_git);
277 }
278
279 sub do_expand_trees {
280         my $repository = shift;
281         my @missing = @_;
282
283         my @cmd = ("expand-trees", "--mainline=$repository",
284                 "--suse=$suse_path", "--append");
285         push(@cmd, "--force") if $force;
286         system(@cmd, @missing);
287         if ($? != 0) {
288                 die "expand-trees failed\n";
289         }
290 }
291
292 # fill the %suse_tree2expanded array
293 sub expand_trees {
294         load_expanded_trees();
295         my @missing = sort(grep { !defined($suse_tree2expanded{$_}{tree}) }
296                 keys(%suse_tree2expanded));
297         return unless @missing;
298         print STDERR "Need to expand ", scalar(@missing), " trees\n";
299         if (!$jobs || $jobs == 1) {
300                 do_expand_trees($mainline_path, @missing);
301         } else {
302                 for (my $i = 0; $i < $jobs; $i++) {
303                         my $clone = "$mainline_path-$i";
304                         system("rm", "-rf", "$clone");
305                         my $start = int($i * $#missing / $jobs);
306                         if ($start) {
307                                 $start++;
308                         }
309                         my $stop = int(($i + 1) * $#missing / $jobs);
310                         next if $stop < $start;
311                         my $pid = fork();
312                         die "Can't fork: $!\n" unless defined($pid);
313                         next if $pid > 0;
314                         system("git", "clone", "-s", "$mainline_path", "$clone");
315                         do_expand_trees($clone, @missing[$start..$stop]);
316                         exit 0;
317                 }
318                 my $pid;
319                 do {
320                         $pid = wait();
321                         if ($pid == 0 && $? != 0) {
322                                 die "job failed\n";
323                         }
324                 } while ($pid > 0);
325                 # merge the expanded-trees branches together
326                 my @ids;
327                 my $id = $mainline_git->read_cmd("rev-parse", "expanded-trees");
328                 if ($id) {
329                         chomp($id);
330                         push(@ids, $id);
331                 }
332                 for (my $i = 0; $i < $jobs; $i++) {
333                         my $clone = "$mainline_path-$i";
334                         next unless -d $clone;
335                         my $git = SUSE::Kernel::Git->new($clone);
336                         my $id = $git->read_cmd("rev-parse", "expanded-trees");
337                         next unless $id;
338                         chomp($id);
339                         system("rsync", "-a", "--exclude=info",
340                                 "$clone/.git/objects/",
341                                 "$mainline_path/.git/objects/");
342                         push(@ids, $id);
343                         print STDERR "job$i: $id\n";
344                 }
345                 chomp(@ids);
346                 my ($in, $out);
347                 ($pid, $in, $out) = $mainline_git->popen2("hash-object",
348                         "-t", "commit", "-w", "--stdin");
349                 # use the empty tree for the merge commit
350                 print $out "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n";
351                 foreach $id (@ids) {
352                         print $out "parent $id\n";
353                 }
354                 print $out "author x <y\@z> 1273225547 +0200\n";
355                 print $out "committer x <y\@z> 1273225547 +0200\n";
356                 print $out "\n";
357                 print $out "merge\n";
358                 close($out);
359                 my $res = <$in>;
360                 chomp($res);
361                 close($in);
362                 waitpid($pid, 0);
363                 die "Error merging expanded-trees branches\n" unless $? == 0 && $res;
364                 $mainline_git->read_cmd("update-ref", "refs/heads/expanded-trees", $res);
365         }
366         $mainline_git->read_cmd("gc", "--auto");
367         load_expanded_trees();
368         @missing = grep { !defined($suse_tree2expanded{$_}{tree}) }
369                 keys(%suse_tree2expanded);
370         if (@missing) {
371                 die "Internal error, following trees were not expanded: ",
372                         join(" ", @missing);
373         }
374 }
375
376 sub mainline_commit {
377         my $version = shift;
378
379         if (!exists($mainline_commits->{$version})) {
380                 $mainline_commits->{$version} =
381                         qx(./mainline-commit --git $mainline_path $version);
382                 die "Unable to determine commit id of v$version"
383                         unless $? == 0 && defined($mainline_commits->{$version});
384                 chomp $mainline_commits->{$version};
385         }
386         return $mainline_commits->{$version};
387 }
388
389 sub expand_commit {
390         my ($commit) = @_;
391
392         if (exists($expanded_commits->{$commit})) {
393                 my $res = $expanded_commits->{$commit};
394                 if (!exists($revs{$commit})) {
395                         # fill data about this commit into %revs so that we
396                         # see it when expanding the children later
397                         my $tree = $mainline_git->read_cmd("rev-parse", "$res:");
398                         # We need a tree object without the BROKEN marker,
399                         # so that we find that id in refs/heads/expanded-trees
400                         $tree = unmark_tree_broken($tree);
401                         $revs{$commit} = {
402                                 expanded => $expanded_tree_data{$tree},
403                                 # For broken trees, we lie here, but that is
404                                 # not a broblem. If the child is broken, it
405                                 # think that the parent is not and add the
406                                 # BROKEN itself.
407                                 final_tree => $tree,
408                         };
409
410                 }
411                 return $res;
412         }
413         return if $exclude_commits{$commit};
414         my $data = $revs{$commit};
415         my @expanded_parents;
416         my ($version_changed, $version_same);
417         my $broken = $data->{expanded}{broken};
418         my $this_version = $data->{expanded}{version};
419         for my $parent (@{$data->{parent}}) {
420                 next if $exclude_commits{$parent};
421                 push(@expanded_parents, expand_commit($parent));
422                 next if $broken or $revs{$parent}->{expanded}{broken};
423                 if ($revs{$parent}->{expanded}{version} eq $this_version) {
424                         $version_same = 1;
425                 } else {
426                         $version_changed = 1;
427                 }
428         }
429         # Consider a commit a version bump iff it is the root commit or
430         # there is a parent with a different version and no parent with
431         # the same version (i.e. broken commits are not taken into account).
432         if ($version_changed && !$version_same ||
433                                         !@expanded_parents && !$broken)  {
434                 my $mainline = mainline_commit($this_version);
435                 if ($mainline !~ /^0*$/) {
436                         push(@expanded_parents, $mainline);
437                 }
438         }
439         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-t",
440                 "commit", "-w", "--stdin");
441         die "Could not expand tree $data->{tree} (commit $commit)\n"
442                 unless $data->{expanded}{tree};
443         if ($broken) {
444                 my ($parent_tree, $parent_broken);
445                 # take the tree of the first parent
446                 for my $parent (@{$data->{parent}}) {
447                         $parent_tree = $revs{$parent}->{final_tree};
448                         $parent_broken = $revs{$parent}->{expanded}{broken};
449                         last;
450                 }
451                 if (!$parent_broken) {
452                         $parent_tree = mark_tree_broken($parent_tree);
453                 }
454                 # We store the final tree in each member of the %revs hash,
455                 # because for broken trees, the final tree is dependent on the
456                 # parent (transitively), so we can't use ->{expanded}{tree}
457                 $data->{final_tree} = $parent_tree;
458         } else {
459                 $data->{final_tree} = $data->{expanded}{tree};
460         }
461         sub translate_id {
462                 my $id = shift;
463
464                 my @match = grep { /^$id/ } keys(%$expanded_commits);
465                 return $id unless @match == 1;
466                 return substr($expanded_commits->{$match[0]}, 0, length($id));
467         }
468         my $message = $data->{message};
469         $message =~ s/\b([0-9a-f]{7,})\b/translate_id($1)/eg;
470         print $out "tree $data->{final_tree}\n";
471         for my $parent (@expanded_parents) {
472                 print $out "parent $parent\n";
473         }
474         print $out "author $data->{author}\n";
475         print $out "committer $data->{committer}\n";
476         print $out "\n";
477         print $out $message;
478         print $out "\n";
479         print $out "suse-commit: $commit\n";
480         if ($broken) {
481                 print $out "Note: This patch series did not apply\n";
482         }
483         close($out);
484         my $res = <$in>;
485         close($in);
486         chomp $res;
487         waitpid($pid, 0);
488         die "Could not expand commit $commit\n" unless $? == 0 && $res;
489         $expanded_commits->{$commit} = $res;
490         return $res;
491 }
492
493 sub expand_tag {
494         my ($id) = @_;
495
496         my $data = $suse_git->tag_data($id);
497         my $expanded = expand_object($data->{object}, $data->{type});
498         return unless $expanded;
499         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-t",
500                 "tag", "-w", "--stdin");
501         print $out "object $expanded\n";
502         print $out "type $data->{type}\n";
503         print $out "tag $data->{tag}\n";
504         if ($data->{tagger}) {
505                 print $out "tagger $data->{tagger}\n";
506         }
507         print $out "\n";
508         print $out $data->{message};
509         close($out);
510         my $res = <$in>;
511         close($in);
512         chomp $res;
513         waitpid($pid, 0);
514         die "Could not expand tag $id\n" unless $? == 0 && $res;
515         return $res;
516 }
517
518 sub expand_object {
519         my ($id, $type) = @_;
520
521         return expand_commit($id) if $type eq "commit";
522         return expand_tag($id) if $type eq "tag";
523         die "Cannot expand object $id of type $type\n";
524 }
525
526 sub expand_refs {
527         for my $ref (@refs) {
528                 print STDERR "Expanding $ref->{fullname}... ";
529                 my $mainline_id = expand_object($ref->{id}, $ref->{type});
530                 if (!$mainline_id) {
531                         print STDERR "skipped\n";
532                         next;
533                 }
534                 $mainline_git->read_cmd("update-ref", $ref->{fullname},
535                         $mainline_id);
536                 print STDERR "done\n";
537         }
538 }
539
540 GetOptions(
541         "m|mainline=s" => \$mainline_path,
542         "s|suse=s" => \$suse_path,
543         "f|force" => \$force,
544         "i|incremental" => \$incremental,
545         "a|all" => \$expand_all,
546         "j|jobs=n" => \$jobs,
547         "h|help" => sub { print $USAGE; exit; },
548 ) or die $USAGE;
549
550 if (!$mainline_path || !$suse_path || (!@ARGV && !$expand_all)) {
551         die $USAGE;
552 }
553 $mainline_path =~ s/\/*$//;
554 $mainline_git = SUSE::Kernel::Git->new($mainline_path);
555 $suse_git = SUSE::Kernel::Git->new($suse_path);
556 if (!$suse_git->writable) {
557         print STDERR "Warning: $suse_path is not writable, this will slow down the process.\n";
558 }
559
560 if ($incremental) {
561         $expanded_commits->load();
562 }
563 if ($expand_all) {
564         push(@ARGV, $suse_git->read_cmd("for-each-ref", "--format=%(refname)",
565                         "refs/heads", "refs/tags"));
566 }
567
568 for my $spec (@ARGV) {
569         if ($spec =~ /^\^([^:]*)$/) {
570                 # ^foo exclude
571                 my @exclude = $suse_git->read_cmd("rev-list", $1, "--");
572                 $exclude_commits{$_} = 1 for @exclude;
573                 next;
574         }
575         # [[from]..[to:]]<ref>
576         if ($spec !~ /^(?:([^\.:]+)?\.\.([^\.:]+:)?)?([^:]+)$/) {
577                 die $USAGE;
578         }
579         my ($bottom, $top, $ref) = ($1, $2, $3);
580         if (!$top) {
581                 $top = $suse_git->read_cmd("rev-parse", "--verify", $ref);
582         } else {
583                 $top =~ s/:$//;
584         }
585         my $fullname = $suse_git->read_cmd("rev-parse", "--symbolic-full-name",
586                 $ref);
587         my $type = $suse_git->read_cmd("cat-file", "-t", $ref);
588         if (!$top || !$fullname || !$type) {
589                 die "Ref $ref does not exist\n";
590         }
591         push(@refs, { id => $top, fullname => $fullname, type => $type });
592         if ($bottom) {
593                 my @exclude = $suse_git->read_cmd("rev-list", $bottom, "--");
594                 $exclude_commits{$_} = 1 for @exclude;
595         }
596 }
597
598 print STDERR "Determining commit ids to expand...\n";
599         for my $ref (@refs) {
600                 for my $rev ($suse_git->read_cmd("rev-list", $ref->{id}, "--")) {
601                 next if exists($expanded_commits->{$rev});
602                 next if exists($exclude_commits{$rev});
603                 $revs{$rev} = {};
604         }
605 }
606
607 load_suse_rev_data();
608 $mainline_commits->load();
609 expand_trees();
610 expand_refs();
611 if ($incremental) {
612         $expanded_commits->save();
613 }
614 $mainline_commits->save();
615 $mainline_git->read_cmd("gc", "--auto");
616