3 # -n do not include custom_ scripts
4 # -v verbose mode, shows progress stuff
6 # this version of locles processes not only all required .pl files
7 # but also all parse_html_templated files.
13 use List::Util qw(first);
18 $bindir = "$basedir/bin/mozilla";
19 $dbupdir = "$basedir/sql/Pg-upgrade";
20 $dbupdir2 = "$basedir/sql/Pg-upgrade2";
21 $menufile = "menu.ini";
22 $submitsearch = qr/type\s*=\s*[\"\']?submit/i;
24 %referenced_html_files = ();
29 # -h extended checks on HTML templates
31 foreach $item (@ARGV) {
36 opendir DIR, "$bindir" or die "$!";
37 @progfiles = grep { /\.pl$/ && !/(_|^\.)/ } readdir DIR;
39 @customfiles = grep /_/, readdir DIR;
42 # put customized files into @customfiles
43 @customfiles = () if ($arg{n});
46 @menufiles = ($menufile);
48 opendir DIR, "$basedir" or die "$!";
49 @menufiles = grep { /.*?_$menufile$/ } readdir DIR;
51 unshift @menufiles, $menufile;
54 opendir DIR, $dbupdir or die "$!";
55 @dbplfiles = grep { /\.pl$/ } readdir DIR;
58 opendir DIR, $dbupdir2 or die "$!";
59 @dbplfiles2 = grep { /\.pl$/ } readdir DIR;
62 # slurp the translations in
67 my %old_texts = %{ $self->{texts} };
69 # Read HTML templates.
71 #@htmltemplates = <../../templates/webpages/*/*_master.html>;
72 #foreach $file (@htmltemplates) {
73 # scanhtmlfile($file);
76 map({ handle_file($_, $bindir); } @progfiles);
77 map({ handle_file($_, $dbupdir); } @dbplfiles);
78 map({ handle_file($_, $dbupdir2); } @dbplfiles2);
81 my ($file, $dir) = @_;
82 print "\n$file" if $arg{v};
87 &scanfile("$dir/$file");
89 # scan custom_{module}.pl or {login}_{module}.pl files
90 foreach $customfile (@customfiles) {
91 if ($customfile =~ /_$file/) {
92 if (-f "$dir/$customfile") {
93 &scanfile("$dir/$customfile");
98 # if this is the menu.pl file
99 if ($file eq 'menu.pl') {
100 foreach $item (@menufiles) {
101 &scanmenu("$basedir/$item");
105 if ($file eq 'menunew.pl') {
106 foreach $item (@menufiles) {
107 &scanmenu("$basedir/$item");
108 print "." if $arg{v};
114 eval { require 'missing'; };
117 foreach $text (keys %$missing) {
118 if ($locale{$text} || $htmllocale{$text}) {
119 unless ($self->{texts}{$text}) {
120 $self->{texts}{$text} = $missing->{$text};
125 open FH, ">$file" or die "$! : $file";
127 print FH q|#!/usr/bin/perl
132 foreach $key (sort keys %locale) {
133 if ($self->{texts}{$key}) {
134 $text = $self->{texts}{$key};
139 $text =~ s/\\$/\\\\/;
142 $keytext =~ s/'/\\'/g;
143 $keytext =~ s/\\$/\\\\/;
145 print FH qq| '$keytext'|
146 . (' ' x (27 - length($keytext)))
147 . qq| => '$text',\n|;
155 foreach $key (sort keys %subrt) {
158 $text =~ s/\\$/\\\\/;
159 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '$text',\n|;
162 foreach $key (sort keys %submit) {
163 $text = ($self->{texts}{$key}) ? $self->{texts}{$key} : $key;
165 $text =~ s/\\$/\\\\/;
168 $english_sub =~ s/'/\\'/g;
169 $english_sub =~ s/\\$/\\\\/;
170 $english_sub = lc $key;
172 $translated_sub = lc $text;
173 $english_sub =~ s/( |-|,)/_/g;
174 $translated_sub =~ s/( |-|,)/_/g;
175 print FH qq| '$translated_sub'|
176 . (' ' x (27 - length($translated_sub)))
177 . qq| => '$english_sub',\n|;
189 #foreach $file (@htmltemplates) {
190 # converthtmlfile($file);
195 open FH, ">all" or die "$! : all";
197 print FH q|#!/usr/bin/perl
199 # These are all the texts to build the translations files.
200 # The file has the form of 'english text' => 'foreign text',
201 # you can add the translation in this file or in the 'missing' file
202 # run locales.pl from this directory to rebuild the translation files
207 foreach $key (sort keys %alllocales) {
208 $text = $self->{texts}{$key};
213 $text =~ s/\\$/\\\\/;
222 print FH qq| '$key'| . (' ' x (27 - length($key))) . qq| => '$text',\n|;
234 open FH, ">missing" or die "$! : missing";
236 print FH q|#!/usr/bin/perl
238 # add the missing texts and run locales.pl to rebuild
243 foreach $text (@missing) {
244 print FH qq| '$text'| . (' ' x (27 - length($text))) . qq| => '',\n|;
263 while (($text, $translation) = each %old_texts) {
264 next if ($alllocales{$text});
266 push @lost, { 'text' => $text, 'translation' => $translation };
270 splice @lost, 0, (scalar @lost - 50) if (scalar @lost > 50);
273 print FH "#!/usr/bin/perl\n\n" .
274 "# The last 50 texts that have been removed.\n" .
275 "# This file will be auto-generated by locales.pl. Do not edit it.\n\n" .
278 foreach $entry (@lost) {
279 $entry->{text} =~ s/\'/\\\'/;
280 $entry->{translation} =~ s/\'/\\\'/;
281 print FH " { 'text' => '$entry->{text}', 'translation' => '$entry->{translation}' },\n";
284 print FH ");\n\n1;\n";
288 open(FH, "LANGUAGE");
291 $trlanguage = $language[0];
295 search_unused_htmlfiles();
296 search_translated_htmlfiles_wo_master();
299 $per = sprintf("%.1f", ($count - $notext) / $count * 100);
300 print "\n$trlanguage - ${per}%";
301 print " - $notext missing" if $notext;
308 sub extract_text_between_parenthesis {
309 my ($fh, $line) = @_;
310 my ($inside_string, $pos, $text, $quote_next) = (undef, 0, "", 0);
313 if (length($line) <= $pos) {
315 return ($text, "") unless ($line);
319 my $cur_char = substr($line, $pos, 1);
321 if (!$inside_string) {
322 if ((length($line) >= ($pos + 3)) && (substr($line, $pos, 2)) eq "qq") {
323 $inside_string = substr($line, $pos + 2, 1);
326 } elsif ((length($line) >= ($pos + 2)) &&
327 (substr($line, $pos, 1) eq "q")) {
328 $inside_string = substr($line, $pos + 1, 1);
331 } elsif (($cur_char eq '"') || ($cur_char eq '\'')) {
332 $inside_string = $cur_char;
334 } elsif (($cur_char eq ")") || ($cur_char eq ',')) {
335 return ($text, substr($line, $pos + 1));
343 } elsif ($cur_char eq '\\') {
347 } elsif ($cur_char eq $inside_string) {
348 undef($inside_string);
361 my $dont_include_subs = shift;
362 my $scanned_files = shift;
364 $scanned_files = {} unless ($scanned_files);
365 return if ($scanned_files->{$file});
366 $scanned_files->{$file} = 1;
368 if (!defined $cached{$file}) {
370 return unless (-f "$file");
372 my $fh = new FileHandle;
373 open $fh, "$file" or die "$! : $file";
375 my ($is_submit, $line_no, $sub_line_no) = (0, 0, 0);
380 # is this another file
381 if (/require\s+\W.*\.pl/) {
383 $newfile =~ s/require\s+\W//;
384 $newfile =~ s|bin/mozilla||;
385 # &scanfile("$bindir/$newfile", 0, $scanned_files);
386 $cached{$file}{scan}{"$bindir/$newfile"} = 1;
387 } elsif (/use\s+SL::(.*?);/) {
390 # &scanfile("../../SL/${1}.pm", 1, $scanned_files);
391 $cached{$file}{scannosubs}{"../../SL/${module}.pm"} = 1;
394 # is this a template call?
395 if (/parse_html_template2?\s*\(\s*[\"\']([\w\/]+)\s*[\"\']/) {
396 my $newfile = "$basedir/templates/webpages/$1_master.html";
397 if (/parse_html_template2/) {
398 print "E: " . strip_base($file) . " is still using 'parse_html_template2' for " . strip_base($newfile) . ".\n";
401 # &scanhtmlfile($newfile);
402 # &converthtmlfile($newfile);
403 $cached{$file}{scanh}{$newfile} = 1;
404 print "." if $arg{v};
406 print "W: missing HTML template: " . strip_base($newfile) . " (referenced from " . strip_base($file) . ")\n";
412 next if ($dont_include_subs);
413 ($null, $subrt) = split / +/;
414 # $subrt{$subrt} = 1;
415 $cached{$file}{subr}{$subrt} = 1;
424 my ($null, $country) = split /,/;
425 $country =~ s/^ +[\"\']//;
426 $country =~ s/[\"\'].*//;
432 # is it a submit button before $locale->
433 if (/$submitsearch/) {
435 if ($` !~ /locale->text/) {
437 $sub_line_no = $line_no;
441 my ($found) = /locale->text.*?\(/;
442 my $postmatch = "$'";
446 ($string, $_) = extract_text_between_parenthesis($fh, $postmatch);
449 # if there is no $ in the string record it
450 unless (($string =~ /\$\D.*/) || ("" eq $string)) {
452 # this guarantees one instance of string
453 # $locale{$string} = 1;
454 $cached{$file}{locale}{$string} = 1;
456 # this one is for all the locales
457 # $alllocales{$string} = 1;
458 $cached{$file}{all}{$string} = 1;
460 # is it a submit button before $locale->
462 # $submit{$string} = 1;
463 $cached{$file}{submit}{$string} = 1;
466 } elsif ($postmatch =~ />/) {
470 # exit loop if there are no more locales on this line
471 ($rc) = ($postmatch =~ /locale->text/);
473 if ( ($postmatch =~ />/)
474 || (!$found && ($sub_line_no != $line_no) && />/)) {
484 map { $alllocales{$_} = 1 } keys %{$cached{$file}{all}};
485 map { $locale{$_} = 1 } keys %{$cached{$file}{locale}};
486 map { $submit{$_} = 1 } keys %{$cached{$file}{submit}};
487 map { $subrt{$_} = 1 } keys %{$cached{$file}{subr}};
488 map { &scanfile($_, 0, $scanned_files) } keys %{$cached{$file}{scan}};
489 map { &scanfile($_, 1, $scanned_files) } keys %{$cached{$file}{scannosubs}};
490 map { &scanhtmlfile($_) } keys %{$cached{$file}{scanh}};
492 @referenced_html_files{keys %{$cached{$file}{scanh}}} = (1) x scalar keys %{$cached{$file}{scanh}};
498 my $fh = new FileHandle;
499 open $fh, "$file" or die "$! : $file";
501 my @a = grep m/^\[/, <$fh>;
505 grep { s/(\[|\])//g } @a;
507 foreach my $item (@a) {
508 @b = split /--/, $item;
509 foreach $string (@b) {
511 $locale{$string} = 1;
512 $alllocales{$string} = 1;
521 if (!defined $cached{$_[0]}) {
522 my %plugins = ( 'loaded' => { }, 'needed' => { } );
524 open(IN, $_[0]) || die $_[0];
529 while (my $line = <IN>) {
532 while ($line =~ m/\[\%[^\w]*use[^\w]+(\w+)[^\w]*?\%\]/gi) {
533 $plugins{loaded}->{$1} = 1;
536 while ($line =~ m/\[\%[^\w]*(\w+)\.\w+\(/g) {
538 $plugins{needed}->{$plugin} = 1 if (first { $_ eq $plugin } qw(HTML LxERP JavaScript MultiColumnIterator));
541 while ("" ne $line) {
543 if ($line =~ m|<translate>|i) {
545 if ($` =~ /$submitsearch/) {
548 substr($line, 0, $eom) = "";
555 if ($line =~ m|</translate>|i) {
557 substr($line, 0, $+[0]) = "";
562 # $submit{$text} = 1;
563 $cached{$_[0]}{submit}{$text} = 1;
566 # $alllocales{$text} = 1;
567 $cached{$_[0]}{all}{$text} = 1;
568 # $htmllocale{$text} = 1;
569 $cached{$_[0]}{html}{$text} = 1;
582 foreach my $plugin (keys %{ $plugins{needed} }) {
583 next if ($plugins{loaded}->{$plugin});
584 print "E: " . strip_base($_[0]) . " requires the Template plugin '$plugin', but is not loaded with '[\% USE $plugin \%]'.\n";
587 &converthtmlfile($_[0]);
590 # copy back into global arrays
591 map { $alllocales{$_} = 1 } keys %{$cached{$_[0]}{all}};
592 map { $htmllocales{$_} = 1 } keys %{$cached{$_[0]}{html}};
593 map { $submit{$_} = 1 } keys %{$cached{$_[0]}{submit}};
596 sub converthtmlfile {
602 open(IN, $file) || die;
604 my $langcode = (split("/", getcwd()))[-1];
605 $file =~ s/_master.html$/_${langcode}.html/;
607 open(OUT, ">$file") || die;
611 while (my $line = <IN>) {
618 while ("" ne $line) {
620 if ($line =~ m|<translate>|i) {
622 substr($line, 0, $+[0]) = "";
624 print(OUT "\n") if ("" eq $line);
627 print(OUT "${line}\n");
632 if ($line =~ m|</translate>|i) {
634 substr($line, 0, $+[0]) = "";
637 $alllocales{$text} = 1;
638 $htmllocale{$text} = 1;
639 print(OUT $self->{"texts"}{$text} || $text);
640 print(OUT "\n") if ("" eq $line);
655 sub search_unused_htmlfiles {
656 my @unscanned_dirs = ('../../templates/webpages');
658 while (scalar @unscanned_dirs) {
659 my $dir = shift @unscanned_dirs;
661 foreach my $entry (<$dir/*>) {
663 push @unscanned_dirs, $entry;
665 } elsif (($entry =~ /_master.html$/) && -f $entry && !$referenced_html_files{$entry}) {
666 print "W: unused HTML template: " . strip_base($entry) . "\n";
673 sub search_translated_htmlfiles_wo_master {
674 my @unscanned_dirs = ('../../templates/webpages');
676 while (scalar @unscanned_dirs) {
677 my $dir = shift @unscanned_dirs;
679 foreach my $entry (<$dir/*>) {
681 push @unscanned_dirs, $entry;
683 } elsif (($entry =~ /_[a-z]+\.html$/) && ($entry !~ /_master.html$/) && -f $entry) {
685 $master =~ s/[a-z]+\.html$/master.html/;
687 print "W: translated HTML template without master: " . strip_base($entry) . "\n";
695 my $s = "$_[0]"; # Create a copy of the string.
698 $s =~ s|templates/webpages/||;