Changeset 31111 in osm for applications


Ignore:
Timestamp:
2015-05-02T15:15:29+02:00 (10 years ago)
Author:
stoecker
Message:

major speedup of i18n process, add some new features, split core translations into two parts

Location:
applications/editors/josm/i18n
Files:
2 added
1 deleted
5 edited

Legend:

Unmodified
Added
Removed

  • applications/editors/josm/i18n/build.xml

    r31103 r31111  
    44    <property name="josm.build.dir" value="../core"/>
    55    <property name="josm.presets" value="${josm.build.dir}/data/defaultpresets.xml"/>
    6     <!--<property name="josm.style" value="${josm.build.dir}/styles/standard/elemstyles.xml"/>-->
    76    <property name="plugin.dir" value="../plugins"/>
    8     <!--<property name="validator.tagfile" value="${josm.build.dir}/data/validator/tagchecker.cfg"/>-->
    97    <property name="maps.srcfile" value="http://josm.openstreetmap.de/maps"/>
     8    <property name="wiki.srcfile" value="http://josm.openstreetmap.de/josmfile?mode=en"/>
    109    <property name="surveyor.srcfile" value="${plugin.dir}/surveyor/resources/surveyor.xml"/>
    1110    <property name="i18n.build.dir" value="build"/>
    1211    <property name="i18n.install.dir" value="${josm.build.dir}/data"/>
     12    <property name="i18n_data.install.dir" value="${josm.build.dir}/data_nodist/trans"/>
    1313    <property name="po.build.dir" value="${i18n.build.dir}"/>
    1414    <property name="ant.build.javac.target" value="1.5"/>
    … …  
    1717    <target name="init" description="Loads the Ant gettext and contrib tasks">
    1818        <taskdef name="gettext-extract" classname="org.xnap.commons.ant.gettext.GettextExtractKeysTask" classpath="${gettexttasks.jar}"/>
    19         <taskdef name="gettext-merge" classname="org.xnap.commons.ant.gettext.GettextMergeKeysTask" classpath="${gettexttasks.jar}"/>
    20         <taskdef name="gettext-dist" classname="org.xnap.commons.ant.gettext.GettextDistTask" classpath="${gettexttasks.jar}"/>
     19        <!-- for the <foreach> possibility -->
    2120        <taskdef resource="net/sf/antcontrib/antcontrib.properties" classpath="${antcontrib.jar}"/>
    2221    </target>
    … …  
    2827            <arg line="convpreset.pl ${josm.presets}"/>
    2928        </exec>
    30         <!--<exec executable="perl" output="${i18n.build.dir}/trans_style.java">
    31             <arg line="convstyle.pl ${josm.style}"/>
    32         </exec>-->
    33         <!--<exec executable="perl" failonerror="true" output="${i18n.build.dir}/trans_validator.java">
    34             <arg line="convvalidator.pl ${validator.tagfile}"/>
    35         </exec>-->
    3629        <exec executable="perl" failonerror="true" output="${i18n.build.dir}/trans_maps.java">
    3730            <arg line="convmaps.pl ${maps.srcfile}"/>
    … …  
    4336            <arg line="convplugins.pl ${plugin.dir}/*/build.xml"/>
    4437        </exec>
     38        <exec executable="perl" failonerror="true">
     39            <arg line="convwiki.pl ${wiki.srcfile} ${i18n.build.dir}/josmfiles ${i18n.build.dir}/josmfiles.zip"/>
     40        </exec>
    4541        <copy file="specialmessages.java" todir="${i18n.build.dir}"/>
    4642    </target>
    … …  
    5046            <fileset dir="${i18n.build.dir}" includes="specialmessages.java"/>
    5147            <fileset dir="${i18n.build.dir}" includes="trans_*.java"/>
     48            <!-- <fileset dir="${i18n.build.dir}/josmfiles" includes="trans_*.*"/> disable for now -->
    5249            <fileset dir="${josm.build.dir}/data/validator" includes="*.mapcss"/>
    5350            <fileset dir="${josm.build.dir}/styles/standard" includes="*.mapcss"/>
    … …  
    5552        </gettext-extract>
    5653    </target>
    57     <target name="po/core.pot" description="Extracts message keys from the JOSM core source code" depends="builddir,init">
     54    <target name="po/core.pot" description="Extracts message keys from the JOSM core source code" depends="trans_.java,init">
    5855        <gettext-extract keysFile="core.pot" poDirectory="po" keywords="-k -ktrc:1c,2 -kmarktrc:1c,2 -ktr -kmarktr -ktrn:1,2 -ktrnc:1c,2,3">
    5956            <fileset dir="${josm.build.dir}/src" includes="**/*.java"/>
    6057            <fileset dir="${i18n.build.dir}" includes="specialmessages.java"/>
    61         </gettext-extract>
    62     </target>
    63     <target name="${i18n.build.dir}/core.2.pot" depends="builddir,po/core.pot" description="Duplicates core.pot for de-duplication in data.pot and plugins.pot">
    64         <copy file="po/core.pot" tofile="${i18n.build.dir}/core.2.pot"/>
    65     </target>
    66     <target name="po/data.pot" description="Extracts message keys from data files, e.g., presets, imagery (minus core strings)" depends="trans_.java,po/core.pot,${i18n.build.dir}/core.2.pot,init">
    67         <gettext-extract keysFile="data.raw.pot" poDirectory="${i18n.build.dir}" keywords="-k -ktrc:1c,2 -kmarktrc:1c,2 -ktr -kmarktr -ktrn:1,2 -ktrnc:1c,2,3">
    68             <fileset dir="${i18n.build.dir}" includes="trans_*.java"/>
     58            <fileset dir="${i18n.build.dir}" includes="trans_presets.java"/>
     59            <!-- maps should be in data instead, but that requires language file loading support in trac python code -->
     60            <fileset dir="${i18n.build.dir}" includes="trans_maps.java"/>
    6961            <fileset dir="${josm.build.dir}/data/validator" includes="*.mapcss"/>
    7062            <fileset dir="${josm.build.dir}/styles/standard" includes="*.mapcss"/>
    7163        </gettext-extract>
     64    </target>
     65    <target name="po/data.pot" description="Extracts message keys from data files, e.g., presets, imagery (minus core strings)" depends="trans_.java,po/core.pot,init">
     66        <gettext-extract keysFile="data.raw.pot" poDirectory="${i18n.build.dir}" keywords="-k -ktrc:1c,2 -kmarktrc:1c,2 -ktr -kmarktr -ktrn:1,2 -ktrnc:1c,2,3">
     67            <fileset dir="${i18n.build.dir}" includes="trans_plugins.java"/>
     68            <!-- <fileset dir="${i18n.build.dir}/josmfiles" includes="trans_*.*"/> -->
     69        </gettext-extract>
     70        <copy todir="${po.build.dir}">
     71            <fileset dir="po" includes="core.pot"/>
     72        </copy>
    7273        <exec executable="msgcomm" failonerror="true" output="po/data.pot">
    73             <arg line="--unique po/core.pot ${i18n.build.dir}/core.2.pot ${i18n.build.dir}/data.raw.pot"/>
     74            <arg line="--unique po/core.pot ${po.build.dir}/core.pot ${i18n.build.dir}/data.raw.pot"/>
    7475        </exec>
     76        <delete file="${po.build.dir}/core.pot"/>
     77        <delete file="${po.build.dir}/data.raw.pot"/>
    7578    </target>
    76     <target name="po/plugins.pot" description="Extracts message keys from the plugins source code (minus core strings)" depends="po/core.pot,${i18n.build.dir}/core.2.pot,init">
     79    <target name="po/plugins.pot" description="Extracts message keys from the plugins source code (minus core and data strings)" depends="trans_.java,po/core.pot,po/data.pot,init">
    7780        <gettext-extract keysFile="plugins.raw.pot" poDirectory="${i18n.build.dir}" keywords="-k -ktrc:1c,2 -kmarktrc:1c,2 -ktr -kmarktr -ktrn:1,2 -ktrnc:1c,2,3">
    7881            <fileset dir="${plugin.dir}" includes="**/*.java"/>
     82            <fileset dir="${i18n.build.dir}" includes="trans_surveyor.java"/>
    7983        </gettext-extract>
     84        <copy todir="${po.build.dir}">
     85            <fileset dir="po" includes="core.pot"/>
     86            <fileset dir="po" includes="data.pot"/>
     87        </copy>
    8088        <exec executable="msgcomm" failonerror="true" output="po/plugins.pot">
    81             <arg line="--unique po/core.pot ${i18n.build.dir}/core.2.pot ${i18n.build.dir}/plugins.raw.pot"/>
     89            <arg line="--unique po/core.pot ${po.build.dir}/core.pot po/data.pot ${po.build.dir}/data.pot ${i18n.build.dir}/plugins.raw.pot"/>
    8290        </exec>
     91        <delete file="${po.build.dir}/core.pot"/>
     92        <delete file="${po.build.dir}/data.pot"/>
     93        <delete file="${po.build.dir}/plugins.raw.pot"/>
    8394    </target>
    84     <target name="build" depends="po/josm.pot,init">
    85         <gettext-merge keysFile="josm.pot" poDirectory="po"/>
     95    <target name="build" depends="pot,allpot">
    8696        <antcall target="coretrans"/>
     97        <antcall target="datatrans"/>
    8798        <foreach param="path" target="plugintrans">
    8899            <path>
    … …  
    91102        </foreach>
    92103    </target>
    93     <target name="singleplugintrans" depends="po/josm.pot,init">
     104    <target name="singleplugintrans" depends="init">
    94105        <antcall target="plugintrans">
    95106            <param name="path" value="${plugin}"/>
    96107        </antcall>
    97108    </target>
    98     <target name="buildcore" depends="po/josm.pot,init">
     109    <target name="buildcore" depends="po/core.pot,po/data.pot">
    99110        <antcall target="coretrans"/>
     111        <antcall target="datatrans"/>
    100112    </target>
    101113    <target name="coretrans">
    102         <mkdir dir="${po.build.dir}/core"/>
    103         <gettext-extract keysFile="josm.pot" poDirectory="${po.build.dir}/core" keywords="-k -ktrc:1c,2 -kmarktrc:1c,2 -ktr -kmarktr -ktrn:1,2 -ktrnc:1c,2,3">
    104             <fileset dir="${josm.build.dir}/src" includes="**/*.java"/>
    105             <fileset dir="${i18n.build.dir}" includes="specialmessages.java"/>
    106             <fileset dir="${i18n.build.dir}" includes="trans_*.java"/>
    107             <fileset dir="${josm.build.dir}/data/validator" includes="*.mapcss"/>
    108         </gettext-extract>
    109         <copy todir="${po.build.dir}/core">
    110             <fileset dir="po"/>
    111         </copy>
    112         <gettext-merge keysFile="josm.pot" poDirectory="${po.build.dir}/core"/>
    113114        <exec executable="perl" failonerror="true">
    114             <arg line="i18n.pl ${i18n.install.dir}/ ${po.build.dir}/core/*.po"/>
     115            <arg line="i18n.pl --potfile=po/core.pot --basedir=${i18n.install.dir}/ po/*.po"/>
    115116        </exec>
    116117    </target>
    117     <target name="plugintrans" depends="init">
     118    <target name="datatrans">
     119        <exec executable="perl" failonerror="true">
     120            <arg line="i18n.pl --potfile=po/data.pot --basedir=${i18n_data.install.dir}/ po/*.po"/>
     121        </exec>
     122    </target>
     123    <target name="plugintrans">
    118124        <basename file="${path}" property="dir"/>
    119125        <mkdir dir="${po.build.dir}/plugin_${dir}"/>
    … …  
    124130            <fileset dir="${plugin.dir}/${dir}" includes="**/*.java"/>
    125131            <fileset dir="${po.build.dir}/plugin_${dir}" includes="trans_*.java"/>
     132            <fileset dir="${i18n.build.dir}" includes="trans_${dir}.java"/>
    126133        </gettext-extract>
    127134        <if>
    128135            <available file="${po.build.dir}/plugin_${dir}/josm.pot"/>
    129136            <then>
    130                 <copy todir="${po.build.dir}/plugin_${dir}">
    131                     <fileset dir="po"/>
    132                 </copy>
    133                 <gettext-merge keysFile="josm.pot" poDirectory="${po.build.dir}/plugin_${dir}"/>
    134137                <exec executable="perl" failonerror="true">
    135                     <arg line="i18n.pl ${plugin.dir}/${dir}/data/ ${po.build.dir}/plugin_${dir}/*.po"/>
     138                    <arg line="i18n.pl --potfile=${po.build.dir}/plugin_${dir}/josm.pot --basedir=${plugin.dir}/${dir}/data/ po/*.po"/>
    136139                </exec>
    137140            </then>
    138141        </if>
    139         <delete dir="${po.build.dir}/plugin_${dir}"/>
     142        <delete dir="${po.build.dir}/plugin_${dir}ss"/>
    140143    </target>
    141144    <target name="clean">
    … …  
    149152        </delete>
    150153    </target>
    151     <target name="test">
     154    <target name="test" depends="pot">
    152155        <mkdir dir="${i18n.build.dir}/test"/>
    153156        <exec executable="perl" failonerror="true">
    154             <arg line="i18n.pl ${i18n.build.dir}/test/ po/*.po"/>
     157            <arg line="i18n.pl --potfile=po/josm.pot --basedir=${i18n.build.dir}/test/ po/*.po"/>
    155158        </exec>
    156159    </target>
    157160    <target name="pot" depends="po/josm.pot" />
    158     <target name="transifex" depends="po/core.pot,po/plugins.pot,po/data.pot" />
     161    <target name="allpot" depends="po/core.pot,po/data.pot,po/plugins.pot" />
    159162    <target name="update">
    160163        <exec executable="perl" failonerror="true">

  • applications/editors/josm/i18n/convmaps.pl

    r31030 r31111  
    66use strict;
    77use LWP::Simple;
    8 use encoding 'utf8';
     8use open qw/:std :encoding(utf8)/;
    99
    1010my $item;

  • applications/editors/josm/i18n/convpreset.pl

    r31076 r31111  
    140140    # display_values / values
    141141    my $sp = ($line =~ /delimiter="(.*?)"/) ? $1 : ($type eq "combo" ? ",":";");
    142     my $vals = ($line =~ /display_values="(.*?)"/) ? $1 : ($line =~ /values="(.*?)"/) ? $1 : undef;
     142    my $vals = ($line =~ / display_values="(.*?)"/) ? $1 : ($line =~ /values="(.*?)"/) ? $1 : undef;
    143143    if($vals)
    144144    {
    … …  
    184184    print "\n";
    185185  }
    186   elsif($line =~ /<\/combo/)
     186  elsif($line =~ /<\/(combo|multiselect)/)
    187187  {
    188188    $combo_n = "";
    … …  
    212212     || $comment)
    213213  {
    214     print "// $line\n";
     214    $line =~ s/[ \t]+((?:short)?description) *= *"([^"]+)/*\/ \/* $1 *\/ tr("$2"); \/*/g;
     215    print "/* $line */\n";
    215216  }
    216217  else

  • applications/editors/josm/i18n/convstyle.pl

    r29999 r31111  
    6565  else
    6666  {
     67    $line =~ s/[ \t]+((?:short)?description) *= *"([^"]+)/*\/ \/* $1 *\/ tr("$2"); \/*/g;
    6768    print "/* $line */\n";
    6869  }

  • applications/editors/josm/i18n/i18n.pl

    r31088 r31111  
    22
    33use utf8;
     4use strict;
    45use open qw/:std :encoding(utf8)/;
    56use Term::ReadKey;
    … …  
    78
    89my $waswarn = 0;
    9 my $maxcount = 0;
    1010my $lang_pattern = '([a-z]{2}_[A-Z]{2}|[a-z]{2,3}|[a-z]{2}\@[a-z]+)';
    1111my $lang_pattern_file = '([a-z]{2}_[A-Z]{2}|[a-z]{2,3}|[a-z]{2}-[a-z]+)';
    … …  
    2020}
    2121
    22 sub loadfiles($@)
     22sub loadpot($)
     23{
     24  my ($file) = @_;
     25  my %all = ();
     26  my %keys = ();
     27  die "Could not open file $file." if(!open FILE,"<:utf8",$file);
     28  my %postate = (last => "", type => "");
     29  my $linenum = 0;
     30  print "Reading file $file\n";
     31  while(<FILE>)
     32  {
     33    ++$linenum;
     34    my $fn = "$file:$linenum";
     35    chomp;
     36    if($_ =~ /^#/ || !$_)
     37    {
     38      checkpo(\%postate, \%all, "pot", "line $linenum in $file", \%keys, 1, undef);
     39      $postate{fuzzy} = 1 if ($_ =~ /fuzzy/);
     40    }
     41    elsif($_ =~ /^"(.*)"$/) {$postate{last} .= $1;}
     42    elsif($_ =~ /^(msg.+) "(.*)"$/)
     43    {
     44      my ($n, $d) = ($1, $2);
     45      my $new = !${postate}{fuzzy} && (($n eq "msgid" && $postate{type} ne "msgctxt") || ($n eq "msgctxt"));
     46      checkpo(\%postate, \%all, "pot", "line $linenum in $file", \%keys, $new, undef);
     47      $postate{last} = $d;
     48      $postate{type} = $n;
     49      $postate{src} = $fn if $new;
     50    }
     51    else
     52    {
     53      die "Strange line $linenum in $file: $_.";
     54    }
     55  }
     56  checkpo(\%postate, \%all, "pot", "line $linenum in $file", \%keys, 1, undef);
     57  close(FILE);
     58  return \%all;
     59}
     60
     61sub loadfiles($$@)
    2362{
    2463  my $desc;
    25   my $all;
    26   my ($lang,@files) = @_;
     64  my %all = ();
     65  my %keys = ();
     66  my ($lang,$use,@files) = @_;
    2767  foreach my $file (@files)
    2868  {
    2969    die "Could not open file $file." if(!open FILE,"<:utf8",$file);
    30     my $linenum = 0;
    31 
    32     my $cnt = -1; # don't count translators info
     70
    3371    if($file =~ /\/$lang_pattern\.po$/)
    3472    {
    … …  
    4583        if($_ =~ /^#/ || !$_)
    4684        {
    47           checkpo(\%postate, \%all, $l, "line $linenum in $file", $keys, 1);
     85          checkpo(\%postate, \%all, $l, "line $linenum in $file", \%keys, 1, $use);
    4886          $postate{fuzzy} = 1 if ($_ =~ /fuzzy/);
    4987        }
    … …  
    5290        {
    5391          my ($n, $d) = ($1, $2);
    54           ++$cnt if $n eq "msgid";
    5592          my $new = !${postate}{fuzzy} && (($n eq "msgid" && $postate{type} ne "msgctxt") || ($n eq "msgctxt"));
    56           checkpo(\%postate, \%all, $l, "line $linenum in $file", $keys, $new);
     93          checkpo(\%postate, \%all, $l, "line $linenum in $file", \%keys, $new, $use);
    5794          $postate{last} = $d;
    5895          $postate{type} = $n;
    … …  
    64101        }
    65102      }
    66       checkpo(\%postate, \%all, $l, "line $linenum in $file", $keys, 1);
     103      checkpo(\%postate, \%all, $l, "line $linenum in $file", \%keys, 1, $use);
    67104    }
    68105    else
    … …  
    70107      die "File format not supported for file $file.";
    71108    }
    72     $maxcount = $cnt if $cnt > $maxcount;
    73109    close(FILE);
    74110  }
    … …  
    79115my $alwaysup = 0;
    80116my $noask = 0;
    81 my $conflicts;
     117my %conflicts;
    82118sub copystring($$$$$$$)
    83119{
    … …  
    99135    else
    100136    {
    101 
    102137      my $f = $data->{$en}{_file} || "";
    103138      $f = ($f ? "$f;".$data->{$en}{"_src.$l"} : $data->{$en}{"_src.$l"}) if $data->{$en}{"_src.$l"};
    … …  
    154189}
    155190
    156 sub checkpo($$$$$$)
    157 {
    158   my ($postate, $data, $l, $txt, $keys, $new) = @_;
     191# Check a current state for new data
     192#
     193# @param postate Pointer to current status hash
     194# @param data    Pointer to final data array
     195# @param l       current language
     196# @param txt     output text in case of error, usually file and line number
     197# @param keys    pointer to hash for info keys extracted from the first msgid "" entry
     198# @param new     whether a data set is finish or not yet complete
     199# @param use     hash to strings to use or undef for all strings
     200#
     201sub checkpo($$$$$$$)
     202{
     203  my ($postate, $data, $l, $txt, $keys, $new, $use) = @_;
    159204
    160205  if($postate->{type} eq "msgid") {$postate->{msgid} = $postate->{last};}
    … …  
    167212  if($new)
    168213  {
    169     if((!$postate->{fuzzy}) && $postate->{msgstr} && $postate->{msgid})
     214    my $en = $postate->{context} ?  "___$postate->{context}___$postate->{msgid}" : $postate->{msgid};
     215    if((!$postate->{fuzzy}) && ($l eq "pot" || $postate->{msgstr}) && $postate->{msgid}
     216    && (!$use || $use->{$en}))
    170217    {
    171218      copystring($data, $postate->{msgid}, $l, $postate->{msgstr},$txt,$postate->{context}, 1);
    172       for($i = 1; exists($postate->{"msgstr_$i"}); ++$i)
    173       { copystring($data, $postate->{msgid}, "$l.$i", $postate->{"msgstr_$i"},$txt,$postate->{context}, 1); }
    174       if($postate->{msgid_1})
    175       { copystring($data, $postate->{msgid}, "en.1", $postate->{msgid_1},$txt,$postate->{context}, 1); }
     219      if(!$use || $use->{$en}{"en.1"})
     220      {
     221        for(my $i = 1; exists($postate->{"msgstr_$i"}); ++$i)
     222        { copystring($data, $postate->{msgid}, "$l.$i", $postate->{"msgstr_$i"},$txt,$postate->{context}, 1); }
     223        if($postate->{msgid_1})
     224        { copystring($data, $postate->{msgid}, "en.1", $postate->{msgid_1},$txt,$postate->{context}, 1); }
     225      }
    176226      copystring($data, $postate->{msgid}, "_src.$l", $postate->{src},$txt,$postate->{context}, 1);
    177227    }
    … …  
    259309    $maxlen = $len if $len > $maxlen;
    260310  }
     311  my $maxcount = keys(%{$data});
    261312  foreach my $file (@files)
    262313  {
    … …  
    306357      { }
    307358      my $val;
    308       $eq = 0;
     359      my $eq = 0;
    309360      if($la eq "en")
    310361      {
    … …  
    357408  my %lang;
    358409  my @po;
    359   my $basename = shift @ARGV;
     410  my $potfile;
     411  my $basename = "./";
     412  foreach my $arg (@ARGV)
     413  {
     414    next if $arg !~ /^--/;
     415    if($arg =~ /^--basedir=(.+)$/)
     416    {
     417      $basename = $1;
     418    }
     419    elsif($arg =~ /^--potfile=(.+)$/)
     420    {
     421      $potfile = $1;
     422    }
     423    else
     424    {
     425      die "Unknown argument $arg.";
     426    }
     427  }
    360428  $basename .= "/" if !($basename =~ /[\/\\:]$/);
    361429  foreach my $arg (@ARGV)
    362430  {
     431    next if $arg =~ /^--/;
    363432    foreach my $f (glob $arg)
    364433    {
    … …  
    368437    }
    369438  }
    370   my %data = loadfiles(\%lang,@po);
     439  my %data = loadfiles(\%lang,$potfile ? loadpot($potfile) : undef, @po);
     440
    371441  my @clang;
    372442  foreach my $la (sort keys %lang)
Note: See TracChangeset for help on using the changeset viewer.