3 # -n do not include custom_ scripts
4 # -v verbose mode, shows progress stuff
6 # this version of locles processes not only all required .pl files
7 # but also all parse_html_templated files.
15 use List::Util qw(first);
19 $OUTPUT_AUTOFLUSH = 1;
21 my $basedir = "../..";
22 my $bindir = "$basedir/bin/mozilla";
23 my $dbupdir = "$basedir/sql/Pg-upgrade";
24 my $dbupdir2 = "$basedir/sql/Pg-upgrade2";
25 my $menufile = "menu.ini";
26 my $submitsearch = qr/type\s*=\s*[\"\']?submit/i;
28 my (%referenced_html_files, %locale, %htmllocale, %alllocales, %cached, %submit, %subrt);
42 GetOptions('no-custom-files' => \$opt_n,
43 'check-files' => \$opt_c,
54 pod2usage(-exitstatus => 0, -verbose => 2);
61 opendir DIR, "$bindir" or die "$!";
62 my @progfiles = grep { /\.pl$/ && !/(_custom|^\.)/ } readdir DIR;
64 my @customfiles = grep /_custom/, readdir DIR;
67 # put customized files into @customfiles
72 @menufiles = ($menufile);
74 opendir DIR, "$basedir" or die "$!";
75 @menufiles = grep { /.*?_$menufile$/ } readdir DIR;
77 unshift @menufiles, $menufile;
80 opendir DIR, $dbupdir or die "$!";
81 my @dbplfiles = grep { /\.pl$/ } readdir DIR;
84 opendir DIR, $dbupdir2 or die "$!";
85 my @dbplfiles2 = grep { /\.pl$/ } readdir DIR;
88 # slurp the translations in
106 my %old_texts = %{ $self->{texts} || {} };
108 # Read HTML templates.
110 #@htmltemplates = <../../templates/webpages/*/*_master.html>;
111 #foreach $file (@htmltemplates) {
112 # scanhtmlfile($file);
115 map({ handle_file($_, $bindir); } @progfiles);
116 map({ handle_file($_, $dbupdir); } @dbplfiles);
117 map({ handle_file($_, $dbupdir2); } @dbplfiles2);
120 my ($file, $dir) = @_;
121 print "\n$file" if $opt_v;
126 &scanfile("$dir/$file");
128 # scan custom_{module}.pl or {login}_{module}.pl files
129 foreach my $customfile (@customfiles) {
130 if ($customfile =~ /_$file/) {
131 if (-f "$dir/$customfile") {
132 &scanfile("$dir/$customfile");
137 # if this is the menu.pl file
138 if ($file eq 'menu.pl') {
139 foreach my $item (@menufiles) {
140 &scanmenu("$basedir/$item");
144 if ($file eq 'menunew.pl') {
145 foreach my $item (@menufiles) {
146 &scanmenu("$basedir/$item");
153 foreach my $text (keys %$missing) {
154 if ($locale{$text} || $htmllocale{$text}) {
155 unless ($self->{texts}{$text}) {
156 $self->{texts}{$text} = $missing->{$text};
161 open FH, ">$file" or die "$! : $file";
163 print FH q|#!/usr/bin/perl
168 foreach my $key (sort keys %locale) {
169 my $text = $self->{texts}{$key} || $key;
171 $text =~ s/\\$/\\\\/;
174 $keytext =~ s/'/\\'/g;
175 $keytext =~ s/\\$/\\\\/;
177 print FH qq| '$keytext'|
178 . (' ' x (27 - length($keytext)))
179 . qq| => '$text',\n|;
187 foreach my $key (sort keys %subrt) {
190 $text =~ s/\\$/\\\\/;
191 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '$text',\n|;
194 foreach my $key (sort keys %submit) {
195 my $text = ($self->{texts}{$key}) ? $self->{texts}{$key} : $key;
197 $text =~ s/\\$/\\\\/;
199 my $english_sub = $key;
200 $english_sub =~ s/'/\\'/g;
201 $english_sub =~ s/\\$/\\\\/;
202 $english_sub = lc $key;
204 my $translated_sub = lc $text;
205 $english_sub =~ s/( |-|,)/_/g;
206 $translated_sub =~ s/( |-|,)/_/g;
207 print FH qq| '$translated_sub'|
208 . (' ' x (27 - length($translated_sub)))
209 . qq| => '$english_sub',\n|;
221 #foreach $file (@htmltemplates) {
222 # converthtmlfile($file);
227 open FH, ">all" or die "$! : all";
229 print FH q|#!/usr/bin/perl
231 # These are all the texts to build the translations files.
232 # The file has the form of 'english text' => 'foreign text',
233 # you can add the translation in this file or in the 'missing' file
234 # run locales.pl from this directory to rebuild the translation files
239 foreach my $key (sort keys %alllocales) {
240 my $text = $self->{texts}{$key};
245 $text =~ s/\\$/\\\\/;
254 print FH qq| '$key'| . (' ' x (27 - length($key))) . qq| => '$text',\n|;
266 open FH, ">missing" or die "$! : missing";
268 print FH q|#!/usr/bin/perl
270 # add the missing texts and run locales.pl to rebuild
275 foreach my $text (@missing) {
276 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '',\n|;
288 while (my ($text, $translation) = each %old_texts) {
289 next if ($alllocales{$text});
291 push @lost, { 'text' => $text, 'translation' => $translation };
295 splice @lost, 0, (scalar @lost - 50) if (scalar @lost > 50);
298 print FH "#!/usr/bin/perl\n\n" .
299 "# The last 50 texts that have been removed.\n" .
300 "# This file will be auto-generated by locales.pl. Do not edit it.\n\n" .
303 foreach my $entry (@lost) {
304 $entry->{text} =~ s/\'/\\\'/g;
305 $entry->{translation} =~ s/\'/\\\'/g;
306 print FH " { 'text' => '$entry->{text}', 'translation' => '$entry->{translation}' },\n";
309 print FH ");\n\n1;\n";
313 open(FH, "LANGUAGE");
316 my $trlanguage = $language[0];
320 search_unused_htmlfiles();
321 search_translated_htmlfiles_wo_master();
324 my $per = sprintf("%.1f", ($count - $notext) / $count * 100);
325 print "\n$trlanguage - ${per}%";
326 print " - $notext/$count missing" if $notext;
333 sub extract_text_between_parenthesis {
334 my ($fh, $line) = @_;
335 my ($inside_string, $pos, $text, $quote_next) = (undef, 0, "", 0);
338 if (length($line) <= $pos) {
340 return ($text, "") unless ($line);
344 my $cur_char = substr($line, $pos, 1);
346 if (!$inside_string) {
347 if ((length($line) >= ($pos + 3)) && (substr($line, $pos, 2)) eq "qq") {
348 $inside_string = substr($line, $pos + 2, 1);
351 } elsif ((length($line) >= ($pos + 2)) &&
352 (substr($line, $pos, 1) eq "q")) {
353 $inside_string = substr($line, $pos + 1, 1);
356 } elsif (($cur_char eq '"') || ($cur_char eq '\'')) {
357 $inside_string = $cur_char;
359 } elsif (($cur_char eq ")") || ($cur_char eq ',')) {
360 return ($text, substr($line, $pos + 1));
368 } elsif ($cur_char eq '\\') {
372 } elsif ($cur_char eq $inside_string) {
373 undef($inside_string);
386 my $dont_include_subs = shift;
387 my $scanned_files = shift;
392 $scanned_files = {} unless ($scanned_files);
393 return if ($scanned_files->{$file});
394 $scanned_files->{$file} = 1;
396 if (!defined $cached{$file}) {
398 return unless (-f "$file");
400 my $fh = new FileHandle;
401 open $fh, "$file" or die "$! : $file";
403 my ($is_submit, $line_no, $sub_line_no) = (0, 0, 0);
408 # is this another file
409 if (/require\s+\W.*\.pl/) {
411 $newfile =~ s/require\s+\W//;
412 $newfile =~ s|bin/mozilla||;
413 # &scanfile("$bindir/$newfile", 0, $scanned_files);
414 $cached{$file}{scan}{"$bindir/$newfile"} = 1;
415 } elsif (/use\s+SL::(.*?);/) {
418 # &scanfile("../../SL/${1}.pm", 1, $scanned_files);
419 $cached{$file}{scannosubs}{"../../SL/${module}.pm"} = 1;
422 # is this a template call?
423 if (/parse_html_template2?\s*\(\s*[\"\']([\w\/]+)\s*[\"\']/) {
424 my $newfile = "$basedir/templates/webpages/$1_master.html";
425 if (/parse_html_template2/) {
426 print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base($newfile) . ".\n";
429 # &scanhtmlfile($newfile);
430 # &converthtmlfile($newfile);
431 $cached{$file}{scanh}{$newfile} = 1;
434 print "W: missing HTML template: " . strip_base($newfile) . " (referenced from " . strip_base($file) . ")\n";
440 next if ($dont_include_subs);
441 my $subrt = (split / +/)[1];
442 # $subrt{$subrt} = 1;
443 $cached{$file}{subr}{$subrt} = 1;
452 my ($null, $country) = split /,/;
453 $country =~ s/^ +[\"\']//;
454 $country =~ s/[\"\'].*//;
460 # is it a submit button before $locale->
461 if (/$submitsearch/) {
463 if ($` !~ /locale->text/) {
465 $sub_line_no = $line_no;
469 my ($found) = /locale->text.*?\(/;
470 my $postmatch = "$'";
474 ($string, $_) = extract_text_between_parenthesis($fh, $postmatch);
477 # if there is no $ in the string record it
478 unless (($string =~ /\$\D.*/) || ("" eq $string)) {
480 # this guarantees one instance of string
481 # $locale{$string} = 1;
482 $cached{$file}{locale}{$string} = 1;
484 # this one is for all the locales
485 # $alllocales{$string} = 1;
486 $cached{$file}{all}{$string} = 1;
488 # is it a submit button before $locale->
490 # $submit{$string} = 1;
491 $cached{$file}{submit}{$string} = 1;
494 } elsif ($postmatch =~ />/) {
498 # exit loop if there are no more locales on this line
499 ($rc) = ($postmatch =~ /locale->text/);
501 if ( ($postmatch =~ />/)
502 || (!$found && ($sub_line_no != $line_no) && />/)) {
512 map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}};
513 map { $locale{$_} = 1 } keys %{$cached{$file}{locale}};
514 map { $submit{$_} = 1 } keys %{$cached{$file}{submit}};
515 map { $subrt{$_} = 1 } keys %{$cached{$file}{subr}};
516 map { &scanfile($_, 0, $scanned_files) } keys %{$cached{$file}{scan}};
517 map { &scanfile($_, 1, $scanned_files) } keys %{$cached{$file}{scannosubs}};
518 map { &scanhtmlfile($_) } keys %{$cached{$file}{scanh}};
520 @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}};
526 my $fh = new FileHandle;
527 open $fh, "$file" or die "$! : $file";
529 my @a = grep m/^\[/, <$fh>;
533 grep { s/(\[|\])//g } @a;
535 foreach my $item (@a) {
536 my @b = split /--/, $item;
537 foreach my $string (@b) {
539 $locale{$string} = 1;
540 $alllocales{$string} = 1;
549 if (!defined $cached{$_[0]}) {
550 my %plugins = ( 'loaded' => { }, 'needed' => { } );
552 open(IN, $_[0]) || die $_[0];
557 while (my $line = <IN>) {
560 while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) {
561 $plugins{loaded}->{$1} = 1;
564 while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) {
566 $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript MultiColumnIterator));
569 while ($line =~ m/(?: # Start von Variante 1: LxERP.t8('...'); ohne darumliegende [% ... %]-Tags
570 LxERP\.t8\( # LxERP.t8(
571 [\'\"] # Anfang des zu übersetzenden Strings
572 (.*?) # Der zu übersetzende String
573 [\'\"] # Ende des zu übersetzenden Strings
574 | # Start von Variante 2: [% '...' | $T8 %]
575 \[\% # Template-Start-Tag
576 [\-~#]* # Whitespace-Unterdrückung
577 \s* # Optional beliebig viele Whitespace
578 [\'\"] # Anfang des zu übersetzenden Strings
579 (.*?) # Der zu übersetzende String
580 [\'\"] # Ende des zu übersetzenden Strings
581 \s*\|\s* # Pipe-Zeichen mit optionalen Whitespace davor und danach
583 .*? # Optionale Argumente für den Filter
585 [\-~#]* # Whitespace-Unterdrückung
586 \%\] # Template-Ende-Tag
589 my $string = $1 || $2;
590 print "Found filter >>>$string<<<\n" if $debug;
591 substr $line, $LAST_MATCH_START[1], $LAST_MATCH_END[0] - $LAST_MATCH_START[0], '';
593 $cached{$_[0]}{all}{$string} = 1;
594 $cached{$_[0]}{html}{$string} = 1;
595 $cached{$_[0]}{submit}{$string} = 1 if $PREMATCH =~ /$submitsearch/;
596 $plugins{needed}->{T8} = 1;
599 while ("" ne $line) {
601 if ($line =~ m|<translate>|i) {
603 if ($` =~ /$submitsearch/) {
606 substr($line, 0, $eom) = "";
613 if ($line =~ m|</translate>|i) {
615 substr($line, 0, $+[0]) = "";
620 # $submit{$text} = 1;
621 $cached{$_[0]}{submit}{$text} = 1;
624 # $alllocales{$text} = 1;
625 $cached{$_[0]}{all}{$text} = 1;
626 # $htmllocale{$text} = 1;
627 $cached{$_[0]}{html}{$text} = 1;
640 foreach my $plugin (keys %{ $plugins{needed} }) {
641 next if ($plugins{loaded}->{$plugin});
642 print "E: " . strip_base($_[0]) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n";
645 &converthtmlfile($_[0]);
648 # copy back into global arrays
649 map { $alllocales{$_} = 1 } keys %{$cached{$_[0]}{all}};
650 map { $htmllocale{$_} = 1 } keys %{$cached{$_[0]}{html}};
651 map { $submit{$_} = 1 } keys %{$cached{$_[0]}{submit}};
654 sub converthtmlfile {
660 open(IN, $file) || die;
662 my $langcode = (split("/", getcwd()))[-1];
663 $file =~ s/_master.html$/_${langcode}.html/;
665 open(OUT, ">$file") || die;
669 while (my $line = <IN>) {
676 while ("" ne $line) {
678 if ($line =~ m|<translate>|i) {
680 substr($line, 0, $+[0]) = "";
682 print(OUT "\n") if ("" eq $line);
685 print(OUT "${line}\n");
690 if ($line =~ m|</translate>|i) {
692 substr($line, 0, $+[0]) = "";
695 $alllocales{$text} = 1;
696 $htmllocale{$text} = 1;
697 print(OUT $self->{"texts"}{$text} || $text);
698 print(OUT "\n") if ("" eq $line);
713 sub search_unused_htmlfiles {
714 my @unscanned_dirs = ('../../templates/webpages');
716 while (scalar @unscanned_dirs) {
717 my $dir = shift @unscanned_dirs;
719 foreach my $entry (<$dir/*>) {
721 push @unscanned_dirs, $entry;
723 } elsif (($entry =~ /_master.html$/) && -f $entry && !$referenced_html_files{$entry}) {
724 print "W: unused HTML template: " . strip_base($entry) . "\n";
731 sub search_translated_htmlfiles_wo_master {
732 my @unscanned_dirs = ('../../templates/webpages');
734 while (scalar @unscanned_dirs) {
735 my $dir = shift @unscanned_dirs;
737 foreach my $entry (<$dir/*>) {
739 push @unscanned_dirs, $entry;
741 } elsif (($entry =~ /_[a-z]+\.html$/) && ($entry !~ /_master.html$/) && -f $entry) {
743 $master =~ s/[a-z]+\.html$/master.html/;
745 print "W: translated HTML template without master: " . strip_base($entry) . "\n";
753 my $s = "$_[0]"; # Create a copy of the string.
756 $s =~ s|templates/webpages/||;
765 locales.pl - Collect strings for translation in Lx-Office
772 -n, --no-custom-files Do not process files whose name contains "_"
773 -c, --check-files Run extended checks on HTML files
774 -v, --verbose Be more verbose
775 -h, --help Show this help
781 =item B<-n>, B<--no-custom-files>
783 Do not process files whose name contains "_", e.g. "custom_io.pl".
785 =item B<-c>, B<--check-files>
787 Run extended checks on the usage of templates. This can be used to
788 discover HTML templates that are never used as well as the usage of
789 non-existing HTML templates.
791 =item B<-v>, B<--verbose>
799 This script collects strings from Perl files, the menu.ini file and
800 HTML templates and puts them into the file "all" for translation. It
801 also distributes those translations back to the individual files.