3 # -n do not include custom_ scripts
4 # -v verbose mode, shows progress stuff
6 # this version of locles processes not only all required .pl files
7 # but also all parse_html_templated files.
15 use List::Util qw(first);
19 $OUTPUT_AUTOFLUSH = 1;
21 my $basedir = "../..";
22 my $bindir = "$basedir/bin/mozilla";
23 my $dbupdir = "$basedir/sql/Pg-upgrade";
24 my $dbupdir2 = "$basedir/sql/Pg-upgrade2";
25 my $menufile = "menu.ini";
26 my $submitsearch = qr/type\s*=\s*[\"\']?submit/i;
28 my (%referenced_html_files, %locale, %htmllocale, %alllocales, %cached, %submit, %subrt);
42 GetOptions('no-custom-files' => \$opt_n,
43 'check-files' => \$opt_c,
54 pod2usage(-exitstatus => 0, -verbose => 2);
61 opendir DIR, "$bindir" or die "$!";
62 my @progfiles = grep { /\.pl$/ && !/(_|^\.)/ } readdir DIR;
64 my @customfiles = grep /_/, readdir DIR;
67 # put customized files into @customfiles
72 @menufiles = ($menufile);
74 opendir DIR, "$basedir" or die "$!";
75 @menufiles = grep { /.*?_$menufile$/ } readdir DIR;
77 unshift @menufiles, $menufile;
80 opendir DIR, $dbupdir or die "$!";
81 my @dbplfiles = grep { /\.pl$/ } readdir DIR;
84 opendir DIR, $dbupdir2 or die "$!";
85 my @dbplfiles2 = grep { /\.pl$/ } readdir DIR;
88 # slurp the translations in
106 my %old_texts = %{ $self->{texts} || {} };
108 # Read HTML templates.
110 #@htmltemplates = <../../templates/webpages/*/*_master.html>;
111 #foreach $file (@htmltemplates) {
112 # scanhtmlfile($file);
115 map({ handle_file($_, $bindir); } @progfiles);
116 map({ handle_file($_, $dbupdir); } @dbplfiles);
117 map({ handle_file($_, $dbupdir2); } @dbplfiles2);
120 my ($file, $dir) = @_;
121 print "\n$file" if $opt_v;
126 &scanfile("$dir/$file");
128 # scan custom_{module}.pl or {login}_{module}.pl files
129 foreach my $customfile (@customfiles) {
130 if ($customfile =~ /_$file/) {
131 if (-f "$dir/$customfile") {
132 &scanfile("$dir/$customfile");
137 # if this is the menu.pl file
138 if ($file eq 'menu.pl') {
139 foreach my $item (@menufiles) {
140 &scanmenu("$basedir/$item");
144 if ($file eq 'menunew.pl') {
145 foreach my $item (@menufiles) {
146 &scanmenu("$basedir/$item");
153 foreach my $text (keys %$missing) {
154 if ($locale{$text} || $htmllocale{$text}) {
155 unless ($self->{texts}{$text}) {
156 $self->{texts}{$text} = $missing->{$text};
161 open FH, ">$file" or die "$! : $file";
163 print FH q|#!/usr/bin/perl
168 foreach my $key (sort keys %locale) {
169 my $text = $self->{texts}{$key} || $key;
171 $text =~ s/\\$/\\\\/;
174 $keytext =~ s/'/\\'/g;
175 $keytext =~ s/\\$/\\\\/;
177 print FH qq| '$keytext'|
178 . (' ' x (27 - length($keytext)))
179 . qq| => '$text',\n|;
187 foreach my $key (sort keys %subrt) {
190 $text =~ s/\\$/\\\\/;
191 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '$text',\n|;
194 foreach my $key (sort keys %submit) {
195 my $text = ($self->{texts}{$key}) ? $self->{texts}{$key} : $key;
197 $text =~ s/\\$/\\\\/;
199 my $english_sub = $key;
200 $english_sub =~ s/'/\\'/g;
201 $english_sub =~ s/\\$/\\\\/;
202 $english_sub = lc $key;
204 my $translated_sub = lc $text;
205 $english_sub =~ s/( |-|,)/_/g;
206 $translated_sub =~ s/( |-|,)/_/g;
207 print FH qq| '$translated_sub'|
208 . (' ' x (27 - length($translated_sub)))
209 . qq| => '$english_sub',\n|;
221 #foreach $file (@htmltemplates) {
222 # converthtmlfile($file);
227 open FH, ">all" or die "$! : all";
229 print FH q|#!/usr/bin/perl
231 # These are all the texts to build the translations files.
232 # The file has the form of 'english text' => 'foreign text',
233 # you can add the translation in this file or in the 'missing' file
234 # run locales.pl from this directory to rebuild the translation files
239 foreach my $key (sort keys %alllocales) {
240 my $text = $self->{texts}{$key};
245 $text =~ s/\\$/\\\\/;
254 print FH qq| '$key'| . (' ' x (27 - length($key))) . qq| => '$text',\n|;
266 open FH, ">missing" or die "$! : missing";
268 print FH q|#!/usr/bin/perl
270 # add the missing texts and run locales.pl to rebuild
275 foreach my $text (@missing) {
276 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '',\n|;
288 while (my ($text, $translation) = each %old_texts) {
289 next if ($alllocales{$text});
291 push @lost, { 'text' => $text, 'translation' => $translation };
295 splice @lost, 0, (scalar @lost - 50) if (scalar @lost > 50);
298 print FH "#!/usr/bin/perl\n\n" .
299 "# The last 50 texts that have been removed.\n" .
300 "# This file will be auto-generated by locales.pl. Do not edit it.\n\n" .
303 foreach my $entry (@lost) {
304 $entry->{text} =~ s/\'/\\\'/g;
305 $entry->{translation} =~ s/\'/\\\'/g;
306 print FH " { 'text' => '$entry->{text}', 'translation' => '$entry->{translation}' },\n";
309 print FH ");\n\n1;\n";
313 open(FH, "LANGUAGE");
316 my $trlanguage = $language[0];
320 search_unused_htmlfiles();
321 search_translated_htmlfiles_wo_master();
324 my $per = sprintf("%.1f", ($count - $notext) / $count * 100);
325 print "\n$trlanguage - ${per}%";
326 print " - $notext/$count missing" if $notext;
333 sub extract_text_between_parenthesis {
334 my ($fh, $line) = @_;
335 my ($inside_string, $pos, $text, $quote_next) = (undef, 0, "", 0);
338 if (length($line) <= $pos) {
340 return ($text, "") unless ($line);
344 my $cur_char = substr($line, $pos, 1);
346 if (!$inside_string) {
347 if ((length($line) >= ($pos + 3)) && (substr($line, $pos, 2)) eq "qq") {
348 $inside_string = substr($line, $pos + 2, 1);
351 } elsif ((length($line) >= ($pos + 2)) &&
352 (substr($line, $pos, 1) eq "q")) {
353 $inside_string = substr($line, $pos + 1, 1);
356 } elsif (($cur_char eq '"') || ($cur_char eq '\'')) {
357 $inside_string = $cur_char;
359 } elsif (($cur_char eq ")") || ($cur_char eq ',')) {
360 return ($text, substr($line, $pos + 1));
368 } elsif ($cur_char eq '\\') {
372 } elsif ($cur_char eq $inside_string) {
373 undef($inside_string);
386 my $dont_include_subs = shift;
387 my $scanned_files = shift;
392 $scanned_files = {} unless ($scanned_files);
393 return if ($scanned_files->{$file});
394 $scanned_files->{$file} = 1;
396 if (!defined $cached{$file}) {
398 return unless (-f "$file");
400 my $fh = new FileHandle;
401 open $fh, "$file" or die "$! : $file";
403 my ($is_submit, $line_no, $sub_line_no) = (0, 0, 0);
408 # is this another file
409 if (/require\s+\W.*\.pl/) {
411 $newfile =~ s/require\s+\W//;
412 $newfile =~ s|bin/mozilla||;
413 # &scanfile("$bindir/$newfile", 0, $scanned_files);
414 $cached{$file}{scan}{"$bindir/$newfile"} = 1;
415 } elsif (/use\s+SL::(.*?);/) {
418 # &scanfile("../../SL/${1}.pm", 1, $scanned_files);
419 $cached{$file}{scannosubs}{"../../SL/${module}.pm"} = 1;
422 # is this a template call?
423 if (/parse_html_template2?\s*\(\s*[\"\']([\w\/]+)\s*[\"\']/) {
424 my $newfile = "$basedir/templates/webpages/$1_master.html";
425 if (/parse_html_template2/) {
426 print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base($newfile) . ".\n";
429 # &scanhtmlfile($newfile);
430 # &converthtmlfile($newfile);
431 $cached{$file}{scanh}{$newfile} = 1;
434 print "W: missing HTML template: " . strip_base($newfile) . " (referenced from " . strip_base($file) . ")\n";
440 next if ($dont_include_subs);
441 my $subrt = (split / +/)[1];
442 # $subrt{$subrt} = 1;
443 $cached{$file}{subr}{$subrt} = 1;
452 my ($null, $country) = split /,/;
453 $country =~ s/^ +[\"\']//;
454 $country =~ s/[\"\'].*//;
460 # is it a submit button before $locale->
461 if (/$submitsearch/) {
463 if ($` !~ /locale->text/) {
465 $sub_line_no = $line_no;
469 my ($found) = /locale->text.*?\(/;
470 my $postmatch = "$'";
474 ($string, $_) = extract_text_between_parenthesis($fh, $postmatch);
477 # if there is no $ in the string record it
478 unless (($string =~ /\$\D.*/) || ("" eq $string)) {
480 # this guarantees one instance of string
481 # $locale{$string} = 1;
482 $cached{$file}{locale}{$string} = 1;
484 # this one is for all the locales
485 # $alllocales{$string} = 1;
486 $cached{$file}{all}{$string} = 1;
488 # is it a submit button before $locale->
490 # $submit{$string} = 1;
491 $cached{$file}{submit}{$string} = 1;
494 } elsif ($postmatch =~ />/) {
498 # exit loop if there are no more locales on this line
499 ($rc) = ($postmatch =~ /locale->text/);
501 if ( ($postmatch =~ />/)
502 || (!$found && ($sub_line_no != $line_no) && />/)) {
512 map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}};
513 map { $locale{$_} = 1 } keys %{$cached{$file}{locale}};
514 map { $submit{$_} = 1 } keys %{$cached{$file}{submit}};
515 map { $subrt{$_} = 1 } keys %{$cached{$file}{subr}};
516 map { &scanfile($_, 0, $scanned_files) } keys %{$cached{$file}{scan}};
517 map { &scanfile($_, 1, $scanned_files) } keys %{$cached{$file}{scannosubs}};
518 map { &scanhtmlfile($_) } keys %{$cached{$file}{scanh}};
520 @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}};
526 my $fh = new FileHandle;
527 open $fh, "$file" or die "$! : $file";
529 my @a = grep m/^\[/, <$fh>;
533 grep { s/(\[|\])//g } @a;
535 foreach my $item (@a) {
536 my @b = split /--/, $item;
537 foreach my $string (@b) {
539 $locale{$string} = 1;
540 $alllocales{$string} = 1;
549 if (!defined $cached{$_[0]}) {
550 my %plugins = ( 'loaded' => { }, 'needed' => { } );
552 open(IN, $_[0]) || die $_[0];
557 while (my $line = <IN>) {
560 while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) {
561 $plugins{loaded}->{$1} = 1;
564 while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) {
566 $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript MultiColumnIterator));
569 while ($line =~ m/\[\% # Template-Start-Tag
570 [\-~#]* # Whitespace-Unterdrückung
571 \s* # Optional beliebig viele Whitespace
572 [\'\"] # Anfang des zu übersetzenden Strings
573 (.*?) # Der zu übersetzende String
574 [\'\"] # Ende des zu übersetzenden Strings
575 \s*\|\s* # Pipe-Zeichen mit optionalen Whitespace davor und danach
577 .*? # Optionale Argumente für den Filter und Whitespaces
578 [\-~#]* # Whitespace-Unterdrückung
579 \%\] # Template-Ende-Tag
582 print "Found filter >>>$string<<<\n" if $debug;
583 substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], '';
585 $cached{$_[0]}{all}{$string} = 1;
586 $cached{$_[0]}{html}{$string} = 1;
587 $cached{$_[0]}{submit}{$string} = 1 if $PREMATCH =~ /$submitsearch/;
588 $plugins{needed}->{T8} = 1;
591 while ("" ne $line) {
593 if ($line =~ m|<translate>|i) {
595 if ($` =~ /$submitsearch/) {
598 substr($line, 0, $eom) = "";
605 if ($line =~ m|</translate>|i) {
607 substr($line, 0, $+[0]) = "";
612 # $submit{$text} = 1;
613 $cached{$_[0]}{submit}{$text} = 1;
616 # $alllocales{$text} = 1;
617 $cached{$_[0]}{all}{$text} = 1;
618 # $htmllocale{$text} = 1;
619 $cached{$_[0]}{html}{$text} = 1;
632 foreach my $plugin (keys %{ $plugins{needed} }) {
633 next if ($plugins{loaded}->{$plugin});
634 print "E: " . strip_base($_[0]) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n";
637 &converthtmlfile($_[0]);
640 # copy back into global arrays
641 map { $alllocales{$_} = 1 } keys %{$cached{$_[0]}{all}};
642 map { $htmllocale{$_} = 1 } keys %{$cached{$_[0]}{html}};
643 map { $submit{$_} = 1 } keys %{$cached{$_[0]}{submit}};
646 sub converthtmlfile {
652 open(IN, $file) || die;
654 my $langcode = (split("/", getcwd()))[-1];
655 $file =~ s/_master.html$/_${langcode}.html/;
657 open(OUT, ">$file") || die;
661 while (my $line = <IN>) {
668 while ("" ne $line) {
670 if ($line =~ m|<translate>|i) {
672 substr($line, 0, $+[0]) = "";
674 print(OUT "\n") if ("" eq $line);
677 print(OUT "${line}\n");
682 if ($line =~ m|</translate>|i) {
684 substr($line, 0, $+[0]) = "";
687 $alllocales{$text} = 1;
688 $htmllocale{$text} = 1;
689 print(OUT $self->{"texts"}{$text} || $text);
690 print(OUT "\n") if ("" eq $line);
705 sub search_unused_htmlfiles {
706 my @unscanned_dirs = ('../../templates/webpages');
708 while (scalar @unscanned_dirs) {
709 my $dir = shift @unscanned_dirs;
711 foreach my $entry (<$dir/*>) {
713 push @unscanned_dirs, $entry;
715 } elsif (($entry =~ /_master.html$/) && -f $entry && !$referenced_html_files{$entry}) {
716 print "W: unused HTML template: " . strip_base($entry) . "\n";
723 sub search_translated_htmlfiles_wo_master {
724 my @unscanned_dirs = ('../../templates/webpages');
726 while (scalar @unscanned_dirs) {
727 my $dir = shift @unscanned_dirs;
729 foreach my $entry (<$dir/*>) {
731 push @unscanned_dirs, $entry;
733 } elsif (($entry =~ /_[a-z]+\.html$/) && ($entry !~ /_master.html$/) && -f $entry) {
735 $master =~ s/[a-z]+\.html$/master.html/;
737 print "W: translated HTML template without master: " . strip_base($entry) . "\n";
745 my $s = "$_[0]"; # Create a copy of the string.
748 $s =~ s|templates/webpages/||;
757 locales.pl - Collect strings for translation in Lx-Office
764 -n, --no-custom-files Do not process files whose name contains "_"
765 -c, --check-files Run extended checks on HTML files
766 -v, --verbose Be more verbose
767 -h, --help Show this help
773 =item B<-n>, B<--no-custom-files>
775 Do not process files whose name contains "_", e.g. "custom_io.pl".
777 =item B<-c>, B<--check-files>
779 Run extended checks on the usage of templates. This can be used to
780 discover HTML templates that are never used as well as the usage of
781 non-existing HTML templates.
783 =item B<-v>, B<--verbose>
791 This script collects strings from Perl files, the menu.ini file and
792 HTML templates and puts them into the file "all" for translation. It
793 also distributes those translations back to the individual files.