X-Git-Url: http://wagnertech.de/git?a=blobdiff_plain;f=scripts%2Flocales.pl;h=93b34fda183a0ba67ec6fa854f9a377125cc1271;hb=61cdba5d566357f3beabe0e7f3f0cb2d7bdccd73;hp=f03e5c667ad2004b7b6c684f4de2370c0b6083b2;hpb=d18df453f55190d4807af324725ea82a8835d7fc;p=kivitendo-erp.git diff --git a/scripts/locales.pl b/scripts/locales.pl index f03e5c667..93b34fda1 100755 --- a/scripts/locales.pl +++ b/scripts/locales.pl @@ -9,15 +9,17 @@ use utf8; use strict; +use Carp; +use Cwd; use Data::Dumper; use English; +use File::Slurp qw(slurp); use FileHandle; use Getopt::Long; +use IO::Dir; use List::Util qw(first); use POSIX; use Pod::Usage; -use Carp; -use File::Slurp qw(slurp); $OUTPUT_AUTOFLUSH = 1; @@ -28,59 +30,81 @@ my $debug = 0; parse_args(); +my $locale; my $basedir = "../.."; my $locales_dir = "."; my $bindir = "$basedir/bin/mozilla"; -my @progdirs = ( "$basedir/SL/Controller", "$basedir/SL/Template/Plugin", "$basedir/SL/Auth" ); +my @progdirs = ( "$basedir/SL" ); my $dbupdir = "$basedir/sql/Pg-upgrade"; my $dbupdir2 = "$basedir/sql/Pg-upgrade2"; my $menufile = "menu.ini"; +my @javascript_dirs = ($basedir .'/js', $basedir .'/templates/webpages'); +my $javascript_output_dir = $basedir .'/js'; my $submitsearch = qr/type\s*=\s*[\"\']?submit/i; +our $self = {}; +our $missing = {}; +our @lost = (); -my (%referenced_html_files, %locale, %htmllocale, %alllocales, %cached, %submit); +my (%referenced_html_files, %locale, %htmllocale, %alllocales, %cached, %submit, %jslocale); my ($ALL_HEADER, $MISSING_HEADER, $LOST_HEADER); init(); -opendir DIR, "$bindir" or die "$!"; -my @progfiles = map { [ $_, $bindir ] } grep { /\.pl$/ && !/(_custom|^\.)/ } readdir DIR; -seekdir DIR, 0; -my @customfiles = grep /_custom/, readdir DIR; -closedir DIR; +sub find_files { + my ($top_dir_name) = @_; + + my (@files, $finder); + + $finder = sub { + my ($dir_name) = @_; + + tie my %dir_h, 'IO::Dir', $dir_name; + + push @files, grep { -f } map { "${dir_name}/${_}" } keys %dir_h; + my @sub_dirs = grep { -d } map { "${dir_name}/${_}" } grep { ! m/^\.\.?$/ } keys %dir_h; + + $finder->($_) for @sub_dirs; + }; + + $finder->($top_dir_name); -foreach my $dir (@progdirs) { - opendir DIR, $dir or die "$!"; - push @progfiles, map { [ $_, $dir ] } grep { /\.pm$/ } readdir DIR; - closedir DIR; + return @files; } +sub merge_texts { +# overwrite existing entries with the ones from 'missing' + $self->{texts}->{$_} = $missing->{$_} for grep { $missing->{$_} } keys %alllocales; + + # try to set missing entries from lost ones + my %lost_by_text = map { ($_->{text} => $_->{translation}) } @lost; + $self->{texts}->{$_} = $lost_by_text{$_} for grep { !$self->{texts}{$_} } keys %alllocales; +} + +my @bindir_files = find_files($bindir); +my @progfiles = map { m:^(.+)/([^/]+)$:; [ $2, $1 ] } grep { /\.pl$/ && !/_custom/ } @bindir_files; +my @customfiles = grep /_custom/, @bindir_files; + +push @progfiles, map { m:^(.+)/([^/]+)$:; [ $2, $1 ] } grep { /\.pm$/ } map { find_files($_) } @progdirs; + # put customized files into @customfiles -my @menufiles; +my (@menufiles, %dir_h); if ($opt_n) { @customfiles = (); @menufiles = ($menufile); } else { - opendir DIR, "$basedir" or die "$!"; - @menufiles = grep { /.*?_$menufile$/ } readdir DIR; - closedir DIR; - unshift @menufiles, $menufile; + tie %dir_h, 'IO::Dir', $basedir; + @menufiles = map { "$basedir/$_" } grep { /.*?_$menufile$/ } keys %dir_h; + unshift @menufiles, "$basedir/$menufile"; } -opendir DIR, $dbupdir or die "$!"; -my @dbplfiles = grep { /\.pl$/ } readdir DIR; -closedir DIR; +tie %dir_h, 'IO::Dir', $dbupdir; +my @dbplfiles = grep { /\.pl$/ } keys %dir_h; -opendir DIR, $dbupdir2 or die "$!"; -my @dbplfiles2 = grep { /\.pl$/ } readdir DIR; -closedir DIR; +tie %dir_h, 'IO::Dir', $dbupdir2; +my @dbplfiles2 = grep { /\.pl$/ } keys %dir_h; # slurp the translations in -our $self = {}; -our $missing = {}; -our @missing = (); -our @lost = (); - if (-f "$locales_dir/all") { require "$locales_dir/all"; } @@ -98,9 +122,17 @@ chomp $charset; my %old_texts = %{ $self->{texts} || {} }; -map({ handle_file(@{ $_ }); } @progfiles); -map({ handle_file($_, $dbupdir); } @dbplfiles); -map({ handle_file($_, $dbupdir2); } @dbplfiles2); +handle_file(@{ $_ }) for @progfiles; +handle_file($_, $dbupdir) for @dbplfiles; +handle_file($_, $dbupdir2) for @dbplfiles2; +scanmenu($_) for @menufiles; + +for my $file_name (map({find_files($_)} @javascript_dirs)) { + scan_javascript_file($file_name); +} + +# merge entries to translate with entries from files 'missing' and 'lost' +merge_texts(); # generate all generate_file( @@ -110,15 +142,43 @@ generate_file( data_sub => sub { _print_line($_, $self->{texts}{$_}, @_) for sort keys %alllocales }, ); +open(my $js_file, '>:encoding(utf8)', $javascript_output_dir .'/locale/'. $locale .'.js') || die; +print $js_file 'namespace("kivi").setupLocale({'; +my $first_entry = 1; +for my $key (sort(keys(%jslocale))) { + print $js_file ((!$first_entry ? ',' : '') ."\n". _double_quote($key) .':'. _double_quote($self->{texts}{$key})); + $first_entry = 0; +} +print $js_file ("\n"); +print $js_file ('});'."\n"); +close($js_file); + + foreach my $text (keys %$missing) { + if ($locale{$text} || $htmllocale{$text}) { + unless ($self->{texts}{$text}) { + $self->{texts}{$text} = $missing->{$text}; + } + } + } + + # calc and generate missing -push @missing, grep { !$self->{texts}{$_} } sort keys %alllocales; +my @new_missing = grep { !$self->{texts}{$_} } sort keys %alllocales; + +if (@new_missing) { + if ($opt_c) { + my %existing_lc = map { (lc $_ => $_) } grep { $self->{texts}->{$_} } keys %{ $self->{texts} }; + foreach my $entry (@new_missing) { + my $other = $existing_lc{lc $entry}; + print "W: No entry for '${entry}' exists, but there is one with different case: '${other}'\n" if $other; + } + } -if (@missing) { generate_file( file => "$locales_dir/missing", header => $MISSING_HEADER, data_name => '$missing', - data_sub => sub { _print_line($_, '', @_) for @missing }, + data_sub => sub { _print_line($_, '', @_) for @new_missing }, ); } @@ -147,7 +207,7 @@ chomp $trlanguage; search_unused_htmlfiles() if $opt_c; my $count = scalar keys %alllocales; -my $notext = scalar @missing; +my $notext = scalar @new_missing; my $per = sprintf("%.1f", ($count - $notext) / $count * 100); print "\n$trlanguage - ${per}%"; print " - $notext/$count missing" if $notext; @@ -200,6 +260,9 @@ sub parse_args { my $ok = 0; foreach my $dir ("../locale/$arg", "locale/$arg", "../$arg", $arg) { next unless -d $dir && -f "$dir/all" && -f "$dir/LANGUAGE"; + + $locale = $arg; + $ok = chdir $dir; last; } @@ -214,6 +277,9 @@ sub parse_args { . "and no locale directory name was given.\n"; exit 1; } + + $locale ||= (grep { $_ } split m:/:, getcwd())[-1]; + $locale =~ s/\.+$//; } sub handle_file { @@ -233,29 +299,7 @@ sub handle_file { } } - # if this is the menu.pl file - if ($file eq 'menu.pl') { - foreach my $item (@menufiles) { - &scanmenu("$basedir/$item"); - } - } - - if ($file eq 'menunew.pl') { - foreach my $item (@menufiles) { - &scanmenu("$basedir/$item"); - print "." if $opt_v; - } - } - $file =~ s/\.pl//; - - foreach my $text (keys %$missing) { - if ($locale{$text} || $htmllocale{$text}) { - unless ($self->{texts}{$text}) { - $self->{texts}{$text} = $missing->{$text}; - } - } - } } sub extract_text_between_parenthesis { @@ -347,17 +391,32 @@ sub scanfile { $cached{$file}{scannosubs}{"../../SL/${module}.pm"} = 1; } + # Some calls to render() are split over multiple lines. Deal + # with that. + while (/(?:parse_html_template2?|render)\s*\( *$/) { + $_ .= <$fh>; + chomp; + } + # is this a template call? if (/(?:parse_html_template2?|render)\s*\(\s*[\"\']([\w\/]+)\s*[\"\']/) { - my $newfile = "$basedir/templates/webpages/$1.html"; + my $new_file_base = "$basedir/templates/webpages/$1."; if (/parse_html_template2/) { - print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base($newfile) . ".\n"; + print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base("${new_file_base}html") . ".\n"; } - if (-f $newfile) { - $cached{$file}{scanh}{$newfile} = 1; - print "." if $opt_v; - } elsif ($opt_c) { - print "W: missing HTML template: " . strip_base($newfile) . " (referenced from " . strip_base($file) . ")\n"; + + my $found_one = 0; + foreach my $ext (qw(html js json)) { + my $new_file = "${new_file_base}${ext}"; + if (-f $new_file) { + $cached{$file}{scanh}{$new_file} = 1; + print "." if $opt_v; + $found_one = 1; + } + } + + if ($opt_c && !$found_one) { + print "W: missing HTML template: " . strip_base($new_file_base) . "{html,json,js} (referenced from " . strip_base($file) . ")\n"; } } @@ -366,7 +425,7 @@ sub scanfile { while ($rc) { if (/Locale/) { unless (/^use /) { - my ($null, $country) = split /,/; + my ($null, $country) = split(/,/); $country =~ s/^ +[\"\']//; $country =~ s/[\"\'].*//; } @@ -383,7 +442,7 @@ sub scanfile { } } - my ($found) = /locale->text.*?\(/; + my ($found) = / (?: locale->text | \b t8 ) \b .*? \(/x; $postmatch = "$'"; if ($found) { @@ -410,7 +469,7 @@ sub scanfile { } # exit loop if there are no more locales on this line - ($rc) = ($postmatch =~ /locale->text/); + ($rc) = ($postmatch =~ /locale->text | \b t8/x); if ( ($postmatch =~ />/) || (!$found && ($sub_line_no != $line_no) && />/)) { @@ -423,14 +482,15 @@ sub scanfile { } - map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}}; - map { $locale{$_} = 1 } keys %{$cached{$file}{locale}}; - map { $submit{$_} = 1 } keys %{$cached{$file}{submit}}; - map { &scanfile($_, 0, $scanned_files) } keys %{$cached{$file}{scan}}; - map { &scanfile($_, 1, $scanned_files) } keys %{$cached{$file}{scannosubs}}; - map { &scanhtmlfile($_) } keys %{$cached{$file}{scanh}}; + $alllocales{$_} = 1 for keys %{$cached{$file}{all}}; + $locale{$_} = 1 for keys %{$cached{$file}{locale}}; + $submit{$_} = 1 for keys %{$cached{$file}{submit}}; - @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}}; + scanfile($_, 0, $scanned_files) for keys %{$cached{$file}{scan}}; + scanfile($_, 1, $scanned_files) for keys %{$cached{$file}{scannosubs}}; + scanhtmlfile($_) for keys %{$cached{$file}{scanh}}; + + $referenced_html_files{$_} = 1 for keys %{$cached{$file}{scanh}}; } sub scanmenu { @@ -467,96 +527,126 @@ sub scanhtmlfile { my $file = shift; - if (!defined $cached{$file}) { - my %plugins = ( 'loaded' => { }, 'needed' => { } ); - - open(IN, $file) || die $file; + return if defined $cached{$file}; - my $copying = 0; - my $issubmit = 0; - my $text = ""; - while (my $line = ) { - chomp($line); + my %plugins = ( 'loaded' => { }, 'needed' => { } ); - while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) { - $plugins{loaded}->{$1} = 1; - } + if (!open(IN, $file)) { + print "E: template file '$file' not found\n"; + return; + } - while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) { - my $plugin = $1; - $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript MultiColumnIterator L)); - } + my $copying = 0; + my $issubmit = 0; + my $text = ""; + while (my $line = ) { + chomp($line); - while ($line =~ m/(?: # Start von Variante 1: LxERP.t8('...'); ohne darumliegende [% ... %]-Tags - (LxERP\.t8)\( # LxERP.t8( ::Parameter $1:: - ([\'\"]) # Anfang des zu übersetzenden Strings ::Parameter $2:: - (.*?) # Der zu übersetzende String ::Parameter $3:: - (?>>$string<<<\n" if $debug; - substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], ''; - - $string = unescape_template_string($string); - $cached{$file}{all}{$string} = 1; - $cached{$file}{html}{$string} = 1; - $cached{$file}{submit}{$string} = 1 if $PREMATCH =~ /$submitsearch/; - $plugins{needed}->{T8} = 1 if $module eq '$T8'; - $plugins{needed}->{LxERP} = 1 if $module eq 'LxERP.t8'; - } + while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) { + $plugins{loaded}->{$1} = 1; + } - while ($line =~ m/\[\% # Template-Start-Tag - [\-~#]? # Whitespace-Unterdrückung - \s* # Optional beliebig viele Whitespace - (?: # Die erkannten Template-Direktiven - PROCESS - | - INCLUDE - ) - \s+ # Mindestens ein Whitespace - [\'\"]? # Anfang des Dateinamens - ([^\s]+) # Beliebig viele Nicht-Whitespaces -- Dateiname - \.html # Endung ".html", ansonsten kann es der Name eines Blocks sein - /ix) { - my $new_file_name = "$basedir/templates/webpages/$1.html"; - $cached{$file}{scanh}{$new_file_name} = 1; - substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], ''; - } + while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) { + my $plugin = $1; + $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript JSON L P)); } - close(IN); + $plugins{needed}->{T8} = 1 if $line =~ m/\[\%.*\|.*\$T8/; + + while ($line =~ m/(?: # Start von Variante 1: LxERP.t8('...'); ohne darumliegende [% ... %]-Tags + (LxERP\.t8)\( # LxERP.t8( ::Parameter $1:: + ([\'\"]) # Anfang des zu übersetzenden Strings ::Parameter $2:: + (.*?) # Der zu übersetzende String ::Parameter $3:: + (?>>$string<<<\n" if $debug; + substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], ''; + + $string = unescape_template_string($string); + $cached{$file}{all}{$string} = 1; + $cached{$file}{html}{$string} = 1; + $cached{$file}{submit}{$string} = 1 if $PREMATCH =~ /$submitsearch/; + $plugins{needed}->{T8} = 1 if $module eq '$T8'; + $plugins{needed}->{LxERP} = 1 if $module eq 'LxERP.t8'; + } - foreach my $plugin (keys %{ $plugins{needed} }) { - next if ($plugins{loaded}->{$plugin}); - print "E: " . strip_base($file) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n"; + while ($line =~ m/\[\% # Template-Start-Tag + [\-~#]? # Whitespace-Unterdrückung + \s* # Optional beliebig viele Whitespace + (?: # Die erkannten Template-Direktiven + PROCESS + | + INCLUDE + ) + \s+ # Mindestens ein Whitespace + [\'\"]? # Anfang des Dateinamens + ([^\s]+) # Beliebig viele Nicht-Whitespaces -- Dateiname + \.html # Endung ".html", ansonsten kann es der Name eines Blocks sein + /ix) { + my $new_file_name = "$basedir/templates/webpages/$1.html"; + $cached{$file}{scanh}{$new_file_name} = 1; + substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], ''; } } + close(IN); + + foreach my $plugin (keys %{ $plugins{needed} }) { + next if ($plugins{loaded}->{$plugin}); + print "E: " . strip_base($file) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n"; + } + # copy back into global arrays - map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}}; - map { $locale{$_} = 1 } keys %{$cached{$file}{html}}; - map { $submit{$_} = 1 } keys %{$cached{$file}{submit}}; + $alllocales{$_} = 1 for keys %{$cached{$file}{all}}; + $locale{$_} = 1 for keys %{$cached{$file}{html}}; + $submit{$_} = 1 for keys %{$cached{$file}{submit}}; - map { scanhtmlfile($_) } keys %{$cached{$file}{scanh}}; + scanhtmlfile($_) for keys %{$cached{$file}{scanh}}; - @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}}; + $referenced_html_files{$_} = 1 for keys %{$cached{$file}{scanh}}; } +sub scan_javascript_file { + my ($file) = @_; + + open(my $fh, $file) || die('can not open file: '. $file); + + while( my $line = readline($fh) ) { + while( $line =~ m/ + kivi.t8 + \s* + \( + \s* + ([\'\"]) + (.*?) + (?