3 # -n do not include custom_ scripts
4 # -v verbose mode, shows progress stuff
6 # this version of locles processes not only all required .pl files
7 # but also all parse_html_templated files.
15 use List::Util qw(first);
19 $OUTPUT_AUTOFLUSH = 1;
21 my $basedir = "../..";
22 my $bindir = "$basedir/bin/mozilla";
23 my $dbupdir = "$basedir/sql/Pg-upgrade";
24 my $dbupdir2 = "$basedir/sql/Pg-upgrade2";
25 my $menufile = "menu.ini";
26 my $submitsearch = qr/type\s*=\s*[\"\']?submit/i;
28 my (%referenced_html_files, %locale, %htmllocale, %alllocales, %cached, %submit, %subrt);
42 GetOptions('no-custom-files' => \$opt_n,
43 'check-files' => \$opt_c,
54 pod2usage(-exitstatus => 0, -verbose => 2);
61 opendir DIR, "$bindir" or die "$!";
62 my @progfiles = grep { /\.pl$/ && !/(_|^\.)/ } readdir DIR;
64 my @customfiles = grep /_/, readdir DIR;
67 # put customized files into @customfiles
72 @menufiles = ($menufile);
74 opendir DIR, "$basedir" or die "$!";
75 @menufiles = grep { /.*?_$menufile$/ } readdir DIR;
77 unshift @menufiles, $menufile;
80 opendir DIR, $dbupdir or die "$!";
81 my @dbplfiles = grep { /\.pl$/ } readdir DIR;
84 opendir DIR, $dbupdir2 or die "$!";
85 my @dbplfiles2 = grep { /\.pl$/ } readdir DIR;
88 # slurp the translations in
106 my %old_texts = %{ $self->{texts} || {} };
108 # Read HTML templates.
110 #@htmltemplates = <../../templates/webpages/*/*_master.html>;
111 #foreach $file (@htmltemplates) {
112 # scanhtmlfile($file);
115 map({ handle_file($_, $bindir); } @progfiles);
116 map({ handle_file($_, $dbupdir); } @dbplfiles);
117 map({ handle_file($_, $dbupdir2); } @dbplfiles2);
120 my ($file, $dir) = @_;
121 print "\n$file" if $opt_v;
126 &scanfile("$dir/$file");
128 # scan custom_{module}.pl or {login}_{module}.pl files
129 foreach my $customfile (@customfiles) {
130 if ($customfile =~ /_$file/) {
131 if (-f "$dir/$customfile") {
132 &scanfile("$dir/$customfile");
137 # if this is the menu.pl file
138 if ($file eq 'menu.pl') {
139 foreach my $item (@menufiles) {
140 &scanmenu("$basedir/$item");
144 if ($file eq 'menunew.pl') {
145 foreach my $item (@menufiles) {
146 &scanmenu("$basedir/$item");
153 foreach my $text (keys %$missing) {
154 if ($locale{$text} || $htmllocale{$text}) {
155 unless ($self->{texts}{$text}) {
156 $self->{texts}{$text} = $missing->{$text};
161 open FH, ">$file" or die "$! : $file";
163 print FH q|#!/usr/bin/perl
168 foreach my $key (sort keys %locale) {
169 my $text = $self->{texts}{$key} || $key;
171 $text =~ s/\\$/\\\\/;
174 $keytext =~ s/'/\\'/g;
175 $keytext =~ s/\\$/\\\\/;
177 print FH qq| '$keytext'|
178 . (' ' x (27 - length($keytext)))
179 . qq| => '$text',\n|;
187 foreach my $key (sort keys %subrt) {
190 $text =~ s/\\$/\\\\/;
191 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '$text',\n|;
194 foreach my $key (sort keys %submit) {
195 my $text = ($self->{texts}{$key}) ? $self->{texts}{$key} : $key;
197 $text =~ s/\\$/\\\\/;
199 my $english_sub = $key;
200 $english_sub =~ s/'/\\'/g;
201 $english_sub =~ s/\\$/\\\\/;
202 $english_sub = lc $key;
204 my $translated_sub = lc $text;
205 $english_sub =~ s/( |-|,)/_/g;
206 $translated_sub =~ s/( |-|,)/_/g;
207 print FH qq| '$translated_sub'|
208 . (' ' x (27 - length($translated_sub)))
209 . qq| => '$english_sub',\n|;
221 #foreach $file (@htmltemplates) {
222 # converthtmlfile($file);
227 open FH, ">all" or die "$! : all";
229 print FH q|#!/usr/bin/perl
231 # These are all the texts to build the translations files.
232 # The file has the form of 'english text' => 'foreign text',
233 # you can add the translation in this file or in the 'missing' file
234 # run locales.pl from this directory to rebuild the translation files
239 foreach my $key (sort keys %alllocales) {
240 my $text = $self->{texts}{$key};
245 $text =~ s/\\$/\\\\/;
254 print FH qq| '$key'| . (' ' x (27 - length($key))) . qq| => '$text',\n|;
266 open FH, ">missing" or die "$! : missing";
268 print FH q|#!/usr/bin/perl
270 # add the missing texts and run locales.pl to rebuild
275 foreach my $text (@missing) {
276 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '',\n|;
288 while (my ($text, $translation) = each %old_texts) {
289 next if ($alllocales{$text});
291 push @lost, { 'text' => $text, 'translation' => $translation };
295 splice @lost, 0, (scalar @lost - 50) if (scalar @lost > 50);
298 print FH "#!/usr/bin/perl\n\n" .
299 "# The last 50 texts that have been removed.\n" .
300 "# This file will be auto-generated by locales.pl. Do not edit it.\n\n" .
303 foreach my $entry (@lost) {
304 $entry->{text} =~ s/\'/\\\'/g;
305 $entry->{translation} =~ s/\'/\\\'/g;
306 print FH " { 'text' => '$entry->{text}', 'translation' => '$entry->{translation}' },\n";
309 print FH ");\n\n1;\n";
313 open(FH, "LANGUAGE");
316 my $trlanguage = $language[0];
320 search_unused_htmlfiles();
321 search_translated_htmlfiles_wo_master();
324 my $per = sprintf("%.1f", ($count - $notext) / $count * 100);
325 print "\n$trlanguage - ${per}%";
326 print " - $notext/$count missing" if $notext;
333 sub extract_text_between_parenthesis {
334 my ($fh, $line) = @_;
335 my ($inside_string, $pos, $text, $quote_next) = (undef, 0, "", 0);
338 if (length($line) <= $pos) {
340 return ($text, "") unless ($line);
344 my $cur_char = substr($line, $pos, 1);
346 if (!$inside_string) {
347 if ((length($line) >= ($pos + 3)) && (substr($line, $pos, 2)) eq "qq") {
348 $inside_string = substr($line, $pos + 2, 1);
351 } elsif ((length($line) >= ($pos + 2)) &&
352 (substr($line, $pos, 1) eq "q")) {
353 $inside_string = substr($line, $pos + 1, 1);
356 } elsif (($cur_char eq '"') || ($cur_char eq '\'')) {
357 $inside_string = $cur_char;
359 } elsif (($cur_char eq ")") || ($cur_char eq ',')) {
360 return ($text, substr($line, $pos + 1));
368 } elsif ($cur_char eq '\\') {
372 } elsif ($cur_char eq $inside_string) {
373 undef($inside_string);
386 my $dont_include_subs = shift;
387 my $scanned_files = shift;
392 $scanned_files = {} unless ($scanned_files);
393 return if ($scanned_files->{$file});
394 $scanned_files->{$file} = 1;
396 if (!defined $cached{$file}) {
398 return unless (-f "$file");
400 my $fh = new FileHandle;
401 open $fh, "$file" or die "$! : $file";
403 my ($is_submit, $line_no, $sub_line_no) = (0, 0, 0);
408 # is this another file
409 if (/require\s+\W.*\.pl/) {
411 $newfile =~ s/require\s+\W//;
412 $newfile =~ s|bin/mozilla||;
413 # &scanfile("$bindir/$newfile", 0, $scanned_files);
414 $cached{$file}{scan}{"$bindir/$newfile"} = 1;
415 } elsif (/use\s+SL::(.*?);/) {
418 # &scanfile("../../SL/${1}.pm", 1, $scanned_files);
419 $cached{$file}{scannosubs}{"../../SL/${module}.pm"} = 1;
422 # is this a template call?
423 if (/parse_html_template2?\s*\(\s*[\"\']([\w\/]+)\s*[\"\']/) {
424 my $newfile = "$basedir/templates/webpages/$1_master.html";
425 if (/parse_html_template2/) {
426 print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base($newfile) . ".\n";
429 # &scanhtmlfile($newfile);
430 # &converthtmlfile($newfile);
431 $cached{$file}{scanh}{$newfile} = 1;
434 print "W: missing HTML template: " . strip_base($newfile) . " (referenced from " . strip_base($file) . ")\n";
440 next if ($dont_include_subs);
441 my $subrt = (split / +/)[1];
442 # $subrt{$subrt} = 1;
443 $cached{$file}{subr}{$subrt} = 1;
452 my ($null, $country) = split /,/;
453 $country =~ s/^ +[\"\']//;
454 $country =~ s/[\"\'].*//;
460 # is it a submit button before $locale->
461 if (/$submitsearch/) {
463 if ($` !~ /locale->text/) {
465 $sub_line_no = $line_no;
469 my ($found) = /locale->text.*?\(/;
470 my $postmatch = "$'";
474 ($string, $_) = extract_text_between_parenthesis($fh, $postmatch);
477 # if there is no $ in the string record it
478 unless (($string =~ /\$\D.*/) || ("" eq $string)) {
480 # this guarantees one instance of string
481 # $locale{$string} = 1;
482 $cached{$file}{locale}{$string} = 1;
484 # this one is for all the locales
485 # $alllocales{$string} = 1;
486 $cached{$file}{all}{$string} = 1;
488 # is it a submit button before $locale->
490 # $submit{$string} = 1;
491 $cached{$file}{submit}{$string} = 1;
494 } elsif ($postmatch =~ />/) {
498 # exit loop if there are no more locales on this line
499 ($rc) = ($postmatch =~ /locale->text/);
501 if ( ($postmatch =~ />/)
502 || (!$found && ($sub_line_no != $line_no) && />/)) {
512 map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}};
513 map { $locale{$_} = 1 } keys %{$cached{$file}{locale}};
514 map { $submit{$_} = 1 } keys %{$cached{$file}{submit}};
515 map { $subrt{$_} = 1 } keys %{$cached{$file}{subr}};
516 map { &scanfile($_, 0, $scanned_files) } keys %{$cached{$file}{scan}};
517 map { &scanfile($_, 1, $scanned_files) } keys %{$cached{$file}{scannosubs}};
518 map { &scanhtmlfile($_) } keys %{$cached{$file}{scanh}};
520 @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}};
526 my $fh = new FileHandle;
527 open $fh, "$file" or die "$! : $file";
529 my @a = grep m/^\[/, <$fh>;
533 grep { s/(\[|\])//g } @a;
535 foreach my $item (@a) {
536 my @b = split /--/, $item;
537 foreach my $string (@b) {
539 $locale{$string} = 1;
540 $alllocales{$string} = 1;
549 if (!defined $cached{$_[0]}) {
550 my %plugins = ( 'loaded' => { }, 'needed' => { } );
552 open(IN, $_[0]) || die $_[0];
557 while (my $line = <IN>) {
560 while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) {
561 $plugins{loaded}->{$1} = 1;
564 while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) {
566 $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript MultiColumnIterator));
569 while ($line =~ m/\[\% # Template-Start-Tag
570 [\-~#]* # Whitespace-Unterdrückung
571 \s* # Optional beliebig viele Whitespace
572 [\'\"] # Anfang des zu übersetzenden Strings
573 (.*?) # Der zu übersetzende String
574 [\'\"] # Ende des zu übersetzenden Strings
575 \s*\|\s* # Pipe-Zeichen mit optionalen Whitespace davor und danach
577 .*? # Optionale Argumente für den Filter und Whitespaces
578 [\-~#]* # Whitespace-Unterdrückung
579 \%\] # Template-Ende-Tag
581 # print "Found filter '$1' in string '$line'\n";
582 $cached{$_[0]}{all}{$1} = 1;
583 $cached{$_[0]}{html}{$1} = 1;
584 $plugins{needed}->{T8} = 1;
585 substr $line, $-[0], $+[0] - $-[0], '';
588 while ("" ne $line) {
590 if ($line =~ m|<translate>|i) {
592 if ($` =~ /$submitsearch/) {
595 substr($line, 0, $eom) = "";
602 if ($line =~ m|</translate>|i) {
604 substr($line, 0, $+[0]) = "";
609 # $submit{$text} = 1;
610 $cached{$_[0]}{submit}{$text} = 1;
613 # $alllocales{$text} = 1;
614 $cached{$_[0]}{all}{$text} = 1;
615 # $htmllocale{$text} = 1;
616 $cached{$_[0]}{html}{$text} = 1;
629 foreach my $plugin (keys %{ $plugins{needed} }) {
630 next if ($plugins{loaded}->{$plugin});
631 print "E: " . strip_base($_[0]) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n";
634 &converthtmlfile($_[0]);
637 # copy back into global arrays
638 map { $alllocales{$_} = 1 } keys %{$cached{$_[0]}{all}};
639 map { $htmllocale{$_} = 1 } keys %{$cached{$_[0]}{html}};
640 map { $submit{$_} = 1 } keys %{$cached{$_[0]}{submit}};
643 sub converthtmlfile {
649 open(IN, $file) || die;
651 my $langcode = (split("/", getcwd()))[-1];
652 $file =~ s/_master.html$/_${langcode}.html/;
654 open(OUT, ">$file") || die;
658 while (my $line = <IN>) {
665 while ("" ne $line) {
667 if ($line =~ m|<translate>|i) {
669 substr($line, 0, $+[0]) = "";
671 print(OUT "\n") if ("" eq $line);
674 print(OUT "${line}\n");
679 if ($line =~ m|</translate>|i) {
681 substr($line, 0, $+[0]) = "";
684 $alllocales{$text} = 1;
685 $htmllocale{$text} = 1;
686 print(OUT $self->{"texts"}{$text} || $text);
687 print(OUT "\n") if ("" eq $line);
702 sub search_unused_htmlfiles {
703 my @unscanned_dirs = ('../../templates/webpages');
705 while (scalar @unscanned_dirs) {
706 my $dir = shift @unscanned_dirs;
708 foreach my $entry (<$dir/*>) {
710 push @unscanned_dirs, $entry;
712 } elsif (($entry =~ /_master.html$/) && -f $entry && !$referenced_html_files{$entry}) {
713 print "W: unused HTML template: " . strip_base($entry) . "\n";
720 sub search_translated_htmlfiles_wo_master {
721 my @unscanned_dirs = ('../../templates/webpages');
723 while (scalar @unscanned_dirs) {
724 my $dir = shift @unscanned_dirs;
726 foreach my $entry (<$dir/*>) {
728 push @unscanned_dirs, $entry;
730 } elsif (($entry =~ /_[a-z]+\.html$/) && ($entry !~ /_master.html$/) && -f $entry) {
732 $master =~ s/[a-z]+\.html$/master.html/;
734 print "W: translated HTML template without master: " . strip_base($entry) . "\n";
742 my $s = "$_[0]"; # Create a copy of the string.
745 $s =~ s|templates/webpages/||;
754 locales.pl - Collect strings for translation in Lx-Office
761 -n, --no-custom-files Do not process files whose name contains "_"
762 -c, --check-files Run extended checks on HTML files
763 -v, --verbose Be more verbose
764 -h, --help Show this help
770 =item B<-n>, B<--no-custom-files>
772 Do not process files whose name contains "_", e.g. "custom_io.pl".
774 =item B<-c>, B<--check-files>
776 Run extended checks on the usage of templates. This can be used to
777 discover HTML templates that are never used as well as the usage of
778 non-existing HTML templates.
780 =item B<-v>, B<--verbose>
788 This script collects strings from Perl files, the menu.ini file and
789 HTML templates and puts them into the file "all" for translation. It
790 also distributes those translations back to the individual files.