gitweb: Handle non UTF-8 text better

[Gitweb] / gitweb.perl

diff --git a/gitweb.perl b/gitweb.perl
index 552b877eda95f78384f1e6be2562410b804e45c0a66d825e99d8df88b472ce5c..47dcc9c9abbb9280bb37960b12a69795e373ac20a868e9a9042b240cdf7b2a8f 100755 (executable)
--- a/gitweb.perl
+++ b/gitweb.perl
@@ -95,6 +95,13 @@ our $default_text_plain_charset  = undef;
 # (relative to the current git repository)
 our $mimetypes_file = undef;
 
 # (relative to the current git repository)
 our $mimetypes_file = undef;
 

+# assume this charset if line contains non-UTF-8 characters;
+# it should be valid encoding (see Encoding::Supported(3pm) for list),
+# for which encoding all byte sequences are valid, for example
+# 'iso-8859-1' aka 'latin1' (it is decoded without checking, so it
+# could be even 'utf-8' for the old behavior)
+our $fallback_encoding = 'latin1';
+

 # You define site-wide feature defaults here; override them with
 # $GITWEB_CONFIG as necessary.
 our %feature = (
 # You define site-wide feature defaults here; override them with
 # $GITWEB_CONFIG as necessary.
 our %feature = (


@@ -133,7 +140,7 @@ our %feature = (
        # $feature{'snapshot'}{'default'} = [undef];
        # To have project specific config enable override in $GITWEB_CONFIG
        # $feature{'snapshot'}{'override'} = 1;
        # $feature{'snapshot'}{'default'} = [undef];
        # To have project specific config enable override in $GITWEB_CONFIG
        # $feature{'snapshot'}{'override'} = 1;

-       # and in project config gitweb.snapshot = none|gzip|bzip2;
+       # and in project config gitweb.snapshot = none|gzip|bzip2|zip;

        'snapshot' => {
                'sub' => \&feature_snapshot,
                'override' => 0,
        'snapshot' => {
                'sub' => \&feature_snapshot,
                'override' => 0,


@@ -245,6 +252,8 @@ sub feature_snapshot {
                return ('x-gzip', 'gz', 'gzip');
        } elsif ($val eq 'bzip2') {
                return ('x-bzip2', 'bz2', 'bzip2');
                return ('x-gzip', 'gz', 'gzip');
        } elsif ($val eq 'bzip2') {
                return ('x-bzip2', 'bz2', 'bzip2');

+       } elsif ($val eq 'zip') {
+               return ('x-zip', 'zip', '');

        } elsif ($val eq 'none') {
                return ();
        }
        } elsif ($val eq 'none') {
                return ();
        }


@@ -601,6 +610,20 @@ sub validate_refname {
        return $input;
 }
 
        return $input;
 }
 

+# decode sequences of octets in utf8 into Perl's internal form,
+# which is utf-8 with utf8 flag set if needed.  gitweb writes out
+# in utf-8 thanks to "binmode STDOUT, ':utf8'" at beginning
+sub to_utf8 {
+       my $str = shift;
+       my $res;
+       eval { $res = decode_utf8($str, Encode::FB_CROAK); };
+       if (defined $res) {
+               return $res;
+       } else {
+               return decode($fallback_encoding, $str, Encode::FB_DEFAULT);
+       }
+}
+

 # quote unsafe chars, but keep the slash, even when it's not
 # correct, but quoted slashes look too horrible in bookmarks
 sub esc_param {
 # quote unsafe chars, but keep the slash, even when it's not
 # correct, but quoted slashes look too horrible in bookmarks
 sub esc_param {


@@ -625,7 +648,7 @@ sub esc_html ($;%) {
        my $str = shift;
        my %opts = @_;
 
        my $str = shift;
        my %opts = @_;
 

-       $str = decode_utf8($str);
+       $str = to_utf8($str);

        $str = $cgi->escapeHTML($str);
        if ($opts{'-nbsp'}) {
                $str =~ s/ / /g;
        $str = $cgi->escapeHTML($str);
        if ($opts{'-nbsp'}) {
                $str =~ s/ / /g;


@@ -639,7 +662,7 @@ sub esc_path {
        my $str = shift;
        my %opts = @_;
 
        my $str = shift;
        my %opts = @_;
 

-       $str = decode_utf8($str);
+       $str = to_utf8($str);

        $str = $cgi->escapeHTML($str);
        if ($opts{'-nbsp'}) {
                $str =~ s/ / /g;
        $str = $cgi->escapeHTML($str);
        if ($opts{'-nbsp'}) {
                $str =~ s/ / /g;


@@ -924,7 +947,7 @@ sub format_subject_html {
 
        if (length($short) < length($long)) {
                return $cgi->a({-href => $href, -class => "list subject",
 
        if (length($short) < length($long)) {
                return $cgi->a({-href => $href, -class => "list subject",

-                               -title => decode_utf8($long)},
+                               -title => to_utf8($long)},

                       esc_html($short) . $extra);
        } else {
                return $cgi->a({-href => $href, -class => "list subject"},
                       esc_html($short) . $extra);
        } else {
                return $cgi->a({-href => $href, -class => "list subject"},


@@ -1238,7 +1261,7 @@ sub git_get_projects_list {
                        if (check_export_ok("$projectroot/$path")) {
                                my $pr = {
                                        path => $path,
                        if (check_export_ok("$projectroot/$path")) {
                                my $pr = {
                                        path => $path,

-                                       owner => decode_utf8($owner),
+                                       owner => to_utf8($owner),

                                };
                                push @list, $pr;
                                (my $forks_path = $path) =~ s/\.git$//;
                                };
                                push @list, $pr;
                                (my $forks_path = $path) =~ s/\.git$//;


@@ -1268,7 +1291,7 @@ sub git_get_project_owner {
                        $pr = unescape($pr);
                        $ow = unescape($ow);
                        if ($pr eq $project) {
                        $pr = unescape($pr);
                        $ow = unescape($ow);
                        if ($pr eq $project) {

-                               $owner = decode_utf8($ow);
+                               $owner = to_utf8($ow);

                                last;
                        }
                }
                                last;
                        }
                }


@@ -1758,7 +1781,7 @@ sub get_file_owner {
        }
        my $owner = $gcos;
        $owner =~ s/[,;].*$//;
        }
        my $owner = $gcos;
        $owner =~ s/[,;].*$//;

-       return decode_utf8($owner);
+       return to_utf8($owner);

 }
 
 ## ......................................................................
 }
 
 ## ......................................................................


@@ -1841,7 +1864,7 @@ sub git_header_html {
 
        my $title = "$site_name";
        if (defined $project) {
 
        my $title = "$site_name";
        if (defined $project) {

-               $title .= " - " . decode_utf8($project);
+               $title .= " - " . to_utf8($project);

                if (defined $action) {
                        $title .= "/$action";
                        if (defined $file_name) {
                if (defined $action) {
                        $title .= "/$action";
                        if (defined $file_name) {


@@ -2115,7 +2138,7 @@ sub git_print_page_path {
 
        print "<div class=\"page_path\">";
        print $cgi->a({-href => href(action=>"tree", hash_base=>$hb),
 
        print "<div class=\"page_path\">";
        print $cgi->a({-href => href(action=>"tree", hash_base=>$hb),

-                     -title => 'tree root'}, decode_utf8("[$project]"));
+                     -title => 'tree root'}, to_utf8("[$project]"));

        print " / ";
        if (defined $name) {
                my @dirname = split '/', $name;
        print " / ";
        if (defined $name) {
                my @dirname = split '/', $name;


@@ -2935,7 +2958,7 @@ sub git_project_list_body {
                ($pr->{'age'}, $pr->{'age_string'}) = @aa;
                if (!defined $pr->{'descr'}) {
                        my $descr = git_get_project_description($pr->{'path'}) || "";
                ($pr->{'age'}, $pr->{'age_string'}) = @aa;
                if (!defined $pr->{'descr'}) {
                        my $descr = git_get_project_description($pr->{'path'}) || "";

-                       $pr->{'descr_long'} = decode_utf8($descr);
+                       $pr->{'descr_long'} = to_utf8($descr);

                        $pr->{'descr'} = chop_str($descr, 25, 5);
                }
                if (!defined $pr->{'owner'}) {
                        $pr->{'descr'} = chop_str($descr, 25, 5);
                }
                if (!defined $pr->{'owner'}) {


@@ -3977,19 +4000,26 @@ sub git_snapshot {
                $hash = git_get_head_hash($project);
        }
 
                $hash = git_get_head_hash($project);
        }
 

-       my $filename = decode_utf8(basename($project)) . "-$hash.tar.$suffix";
+       my $git = git_cmd_str();
+       my $name = $project;
+       $name =~ s/\047/\047\\\047\047/g;
+       my $filename = to_utf8(basename($project));
+       my $cmd;
+       if ($suffix eq 'zip') {
+               $filename .= "-$hash.$suffix";
+               $cmd = "$git archive --format=zip --prefix=\'$name\'/ $hash";
+       } else {
+               $filename .= "-$hash.tar.$suffix";
+               $cmd = "$git archive --format=tar --prefix=\'$name\'/ $hash | $command";
+       }

 
        print $cgi->header(
                -type => "application/$ctype",
                -content_disposition => 'inline; filename="' . "$filename" . '"',
                -status => '200 OK');
 
 
        print $cgi->header(
                -type => "application/$ctype",
                -content_disposition => 'inline; filename="' . "$filename" . '"',
                -status => '200 OK');
 

-       my $git = git_cmd_str();
-       my $name = $project;
-       $name =~ s/\047/\047\\\047\047/g;
-       open my $fd, "-|",
-               "$git archive --format=tar --prefix=\'$name\'/ $hash | $command"
-               or die_error(undef, "Execute git-tar-tree failed");
+       open my $fd, "-|", $cmd
+               or die_error(undef, "Execute git-archive failed");

        binmode STDOUT, ':raw';
        print <$fd>;
        binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi
        binmode STDOUT, ':raw';
        print <$fd>;
        binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi


@@ -5041,7 +5071,8 @@ XML
 
                # get list of changed files
                open my $fd, "-|", git_cmd(), "diff-tree", '-r', @diff_opts,
 
                # get list of changed files
                open my $fd, "-|", git_cmd(), "diff-tree", '-r', @diff_opts,

-                       $co{'parent'}, $co{'id'}, "--", (defined $file_name ? $file_name : ())
+                       $co{'parent'} || "--root",
+                       $co{'id'}, "--", (defined $file_name ? $file_name : ())

                        or next;
                my @difftree = map { chomp; $_ } <$fd>;
                close $fd
                        or next;
                my @difftree = map { chomp; $_ } <$fd>;
                close $fd