Changeset 9505 in josm for trunk


Ignore:
Timestamp:
2016-01-17T16:55:23+01:00 (8 years ago)
Author:
stoecker
Message:

see #12313 - improve groovy script for EII/JOSM difference checks, add checking for best parameter, add output suppression

Location:
trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/build.xml

    r9501 r9505  
    616616    </target>
    617617
     618    <target name="imageryindex">
     619        <taskdef name="groovy" classname="org.codehaus.groovy.ant.Groovy" classpath="${groovy.jar};tools/commons-cli-1.3.1.jar"/>
     620        <echo message="Checking editor imagery difference"/>
     621        <groovy src="scripts/sync_editor_imagery_index.groovy" classpath="dist/josm-custom.jar">
     622            <arg value="-nomissingeii"/>
     623        </groovy>
     624    </target>
     625
     626    <target name="imageryindexdownload">
     627        <exec append="false" executable="wget" failifexecutionfails="true">
     628            <arg value="https://josm.openstreetmap.de/maps"/>
     629            <arg value="-O"/>
     630            <arg value="maps.xml"/>
     631            <arg value="--unlink"/>
     632        </exec>
     633        <exec append="false" executable="wget" failifexecutionfails="true">
     634            <arg value="https://raw.githubusercontent.com/osmlab/editor-imagery-index/gh-pages/imagery.json"/>
     635            <arg value="-O"/>
     636            <arg value="imagery.json"/>
     637            <arg value="--unlink"/>
     638        </exec>
     639        <antcall target="imageryindex"/>
     640    </target>
     641
    618642    <target name="checkstyle">
    619643        <taskdef resource="com/puppycrawl/tools/checkstyle/ant/checkstyle-ant-task.properties"

  • trunk/scripts/sync_editor_imagery_index.groovy

    r9340 r9505  
    3333    List<ImageryInfo> josmEntries;
    3434    JsonArray eiiEntries;
    35    
    36     final static def EII_KNOWN_DUPLICATES = ["http://geolittoral.application.equipement.gouv.fr/wms/metropole?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=ortholittorale&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"]
    37     final static def JOSM_KNOWN_DUPLICATES = ["http://geolittoral.application.equipement.gouv.fr/wms/metropole?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=ortholittorale&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"]
    3835
    3936    def eiiUrls = new HashMap<String, JsonObject>()
    … …  
    4239    static String eiiInputFile = 'imagery.json'
    4340    static String josmInputFile = 'maps.xml'
    44    
     41    static FileWriter outputFile = null
     42    static BufferedWriter outputStream = null
     43    static int skipCount = 0;
     44    static def skipEntries = [:]
     45
    4546    static def options
    4647   
    … …  
    5152        parse_command_line_arguments(args)
    5253        def script = new sync_editor_imagery_index()
     54        script.loadSkip()
    5355        script.loadJosmEntries()
    54         println "*** Loaded ${script.josmEntries.size()} entries (JOSM). ***"
    5556        script.loadEIIEntries()
    56         println "*** Loaded ${script.eiiEntries.size()} entries (EII). ***"
    5757        script.checkInOneButNotTheOther()
    5858        script.checkCommonEntries()
     59        if(outputStream != null) {
     60            outputStream.close();
     61        }
     62        if(outputFile != null) {
     63            outputFile.close();
     64        }
    5965    }
    6066   
    … …  
    6470    static void parse_command_line_arguments(args) {
    6571        def cli = new CliBuilder()
    66         cli._(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
    67         cli._(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
     72        cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
     73        cli.e(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
     74        cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
    6875        cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
     76        cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
     77        cli.m(longOpt:'nomissingeii', argName:"nomissingeii", "don't show missing editor imagery index entries")
    6978        cli.h(longOpt:'help', "show this help")
    7079        options = cli.parse(args)
    … …  
    8089            josmInputFile = options.josm_input
    8190        }
    82     }
    83 
     91        if (options.output && options.output != "-") {
     92            outputFile = new FileWriter(options.output)
     93            outputStream = new BufferedWriter(outputFile)
     94        }
     95    }
     96
     97    void loadSkip() {
     98        if (options.noskip)
     99            return;
     100        skipEntries["+++ EII-URL is not unique: http://geolittoral.application.equipement.gouv.fr/wms/metropole?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=ortholittorale&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
     101        skipEntries["  name differs: http://wms.openstreetmap.fr/tms/1.0.0/tours_2013/{zoom}/{x}/{y}"] = 3
     102        skipEntries["  name differs: http://wms.openstreetmap.fr/tms/1.0.0/tours/{zoom}/{x}/{y}"] = 3
     103        skipEntries["  name differs: https://secure.erlangen.de/arcgiser/services/Luftbilder2011/MapServer/WmsServer?FORMAT=image/bmp&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&LAYERS=Erlangen_ratio10_5cm_gk4.jp2&STYLES=&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 3
     104        skipEntries["  name differs: http://wms.openstreetmap.fr/tms/1.0.0/iomhaiti/{zoom}/{x}/{y}"] = 3
     105        skipEntries["  name differs: http://{switch:a,b,c}.layers.openstreetmap.fr/bano/{zoom}/{x}/{y}.png"] = 3
     106        skipEntries["  name differs: http://ooc.openstreetmap.org/os1/{zoom}/{x}/{y}.jpg"] = 3
     107        skipEntries["  name differs: http://www.gisnet.lv/cgi-bin/osm_latvia?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=piekraste&SRS={proj}&WIDTH={width}&height={height}&BBOX={bbox}"] = 3
     108        skipEntries["  name differs: http://tms.cadastre.openstreetmap.fr/*/tout/{z}/{x}/{y}.png"] = 3
     109        skipEntries["  name differs: http://{switch:a,b,c}.tiles.mapbox.com/v4/enf.e0b8291e/{zoom}/{x}/{y}.png?access_token=pk.eyJ1Ijoib3BlbnN0cmVldG1hcCIsImEiOiJhNVlHd29ZIn0.ti6wATGDWOmCnCYen-Ip7Q"] = 3
     110        skipEntries["  name differs: http://geo.nls.uk/mapdata2/os/25_inch/scotland_1/{zoom}/{x}/{y}.png"] = 3
     111        skipEntries["  name differs: http://geo.nls.uk/mapdata3/os/6_inch_gb_1900/{zoom}/{x}/{y}.png"] = 3
     112        skipEntries["  maxzoom differs: [DE] Bavaria (2 m) - http://geodaten.bayern.de/ogc/ogc_dop200_oa.cgi?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=adv_dop200c&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 3
     113        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries County - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=County&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     114        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries NPWS Reserve - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=NPWSReserve&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     115        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries Parish - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=Parish&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     116        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries Suburb - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=Suburb&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     117        skipEntries["  minzoom differs: [AU] LPI NSW Imagery - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Imagery/MapServer/tile/{zoom}/{y}/{x}"] = 3
     118        skipEntries["  minzoom differs: [AU] LPI NSW Topographic Map - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Topo_Map/MapServer/tile/{zoom}/{y}/{x}"] = 3
     119        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries State Forest - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=StateForest&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     120        skipEntries["  minzoom differs: [AU] LPI NSW Administrative Boundaries LGA - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=LocalGovernmentArea&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
     121        skipEntries["  minzoom differs: [AU] LPI NSW Base Map - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Base_Map/MapServer/tile/{zoom}/{y}/{x}"] = 3
     122    }
     123   
     124    void myprintln(String s) {
     125        if(skipEntries.containsKey(s)) {
     126            skipCount = skipEntries.get(s)
     127        }
     128        if(skipCount) {
     129            skipCount -= 1;
     130            return
     131        }
     132        if(outputStream != null) {
     133            outputStream.write(s);
     134            outputStream.newLine();
     135        } else {
     136            println s;
     137        }
     138    }
     139   
    84140    void loadEIIEntries() {
    85141        FileReader fr = new FileReader(eiiInputFile)
    … …  
    90146        for (def e : eiiEntries) {
    91147            def url = getUrl(e)
    92             if (eiiUrls.containsKey(url) && !EII_KNOWN_DUPLICATES.contains(url))
    93                 throw new Exception("URL is not unique: "+url)
    94             eiiUrls.put(url, e)
    95         }
     148            if (eiiUrls.containsKey(url)) {
     149                myprintln "+++ EII-URL is not unique: "+url
     150            } else {
     151                eiiUrls.put(url, e)
     152            }
     153        }
     154        myprintln "*** Loaded ${eiiEntries.size()} entries (EII). ***"
    96155    }
    97156
    … …  
    102161        for (def e : josmEntries) {
    103162            def url = getUrl(e)
    104             if (josmUrls.containsKey(url) && !JOSM_KNOWN_DUPLICATES.contains(url)) {
    105                 throw new Exception("URL is not unique: "+url)
    106             }
    107             josmUrls.put(url, e)
    108         }
     163            if (josmUrls.containsKey(url)) {
     164                myprintln "+++ JOSM-URL is not unique: "+url
     165            } else {
     166              josmUrls.put(url, e)
     167            }
     168        }
     169        myprintln "*** Loaded ${josmEntries.size()} entries (JOSM). ***"
    109170    }
    110171
    … …  
    122183    void checkInOneButNotTheOther() {
    123184        def l1 = inOneButNotTheOther(eiiUrls, josmUrls)
    124         println "*** URLs found in EII but not in JOSM (${l1.size()}): ***"
     185        myprintln "*** URLs found in EII but not in JOSM (${l1.size()}): ***"
    125186        if (l1.isEmpty()) {
    126             println "  -"
    127         } else {
    128             println Utils.join("\n", l1)
    129         }
    130 
     187            myprintln "  -"
     188        } else {
     189            myprintln Utils.join("\n", l1)
     190        }
     191
     192        if (options.nomissingeii)
     193            return
    131194        def l2 = inOneButNotTheOther(josmUrls, eiiUrls)
    132         println "*** URLs found in JOSM but not in EII (${l2.size()}): ***"
     195        myprintln "*** URLs found in JOSM but not in EII (${l2.size()}): ***"
    133196        if (l2.isEmpty()) {
    134             println "  -"
    135         } else {
    136             println Utils.join("\n", l2)
     197            myprintln "  -"
     198        } else {
     199            myprintln Utils.join("\n", l2)
    137200        }
    138201    }
    139202   
    140203    void checkCommonEntries() {
    141         println "*** Same URL, but different name: ***"
     204        myprintln "*** Same URL, but different name: ***"
    142205        for (def url : eiiUrls.keySet()) {
    143206            def e = eiiUrls.get(url)
    … …  
    145208            def j = josmUrls.get(url)
    146209            if (!getName(e).equals(getName(j))) {
    147                 println "  name differs: $url"
    148                 println "     (IEE):     ${getName(e)}"
    149                 println "     (JOSM):    ${getName(j)}"
    150             }
    151         }
    152        
    153         println "*** Same URL, but different type: ***"
     210                myprintln "  name differs: $url"
     211                myprintln "     (IEE):     ${getName(e)}"
     212                myprintln "     (JOSM):    ${getName(j)}"
     213            }
     214        }
     215       
     216        myprintln "*** Same URL, but different type: ***"
    154217        for (def url : eiiUrls.keySet()) {
    155218            def e = eiiUrls.get(url)
    … …  
    157220            def j = josmUrls.get(url)
    158221            if (!getType(e).equals(getType(j))) {
    159                 println "  type differs: ${getName(j)} - $url"
    160                 println "     (IEE):     ${getType(e)}"
    161                 println "     (JOSM):    ${getType(j)}"
    162             }
    163         }
    164        
    165         println "*** Same URL, but different zoom bounds: ***"
     222                myprintln "  type differs: ${getName(j)} - $url"
     223                myprintln "     (IEE):     ${getType(e)}"
     224                myprintln "     (JOSM):    ${getType(j)}"
     225            }
     226        }
     227       
     228        myprintln "*** Same URL, but different zoom bounds: ***"
    166229        for (def url : eiiUrls.keySet()) {
    167230            def e = eiiUrls.get(url)
    … …  
    172235            Integer jMinZoom = getMinZoom(j)
    173236            if (eMinZoom != jMinZoom) {
    174                 println "  minzoom differs: ${getDescription(j)}"
    175                 println "     (IEE):     ${eMinZoom}"
    176                 println "     (JOSM):    ${jMinZoom}"
     237                myprintln "  minzoom differs: ${getDescription(j)}"
     238                myprintln "     (IEE):     ${eMinZoom}"
     239                myprintln "     (JOSM):    ${jMinZoom}"
    177240            }
    178241            Integer eMaxZoom = getMaxZoom(e)
    179242            Integer jMaxZoom = getMaxZoom(j)
    180243            if (eMaxZoom != jMaxZoom) {
    181                 println "  maxzoom differs: ${getDescription(j)}"
    182                 println "     (IEE):     ${eMaxZoom}"
    183                 println "     (JOSM):    ${jMaxZoom}"
    184             }
    185         }
    186        
    187         println "*** Same URL, but different country code: ***"
     244                myprintln "  maxzoom differs: ${getDescription(j)}"
     245                myprintln "     (IEE):     ${eMaxZoom}"
     246                myprintln "     (JOSM):    ${jMaxZoom}"
     247            }
     248        }
     249       
     250        myprintln "*** Same URL, but different country code: ***"
    188251        for (def url : eiiUrls.keySet()) {
    189252            def e = eiiUrls.get(url)
    … …  
    191254            def j = josmUrls.get(url)
    192255            if (!getCountryCode(e).equals(getCountryCode(j))) {
    193                 println "  country code differs: ${getDescription(j)}"
    194                 println "     (IEE):     ${getCountryCode(e)}"
    195                 println "     (JOSM):    ${getCountryCode(j)}"
     256                myprintln "  country code differs: ${getDescription(j)}"
     257                myprintln "     (IEE):     ${getCountryCode(e)}"
     258                myprintln "     (JOSM):    ${getCountryCode(j)}"
     259            }
     260        }
     261        myprintln "*** Same URL, but different quality: ***"
     262        for (def url : eiiUrls.keySet()) {
     263            def e = eiiUrls.get(url)
     264            if (!josmUrls.containsKey(url)) continue
     265            def j = josmUrls.get(url)
     266            if (!getQuality(e).equals(getQuality(j))) {
     267                myprintln "  quality differs: ${getDescription(j)}"
     268                myprintln "     (IEE):     ${getQuality(e)}"
     269                myprintln "     (JOSM):    ${getQuality(j)}"
    196270            }
    197271        }
    … …  
    241315        return e.getString("country_code", null)
    242316    }
     317    static String getQuality(Object e) {
     318        //if (e instanceof ImageryInfo) return "".equals(e.getQuality()) ? null : e.getQuality()
     319        if (e instanceof ImageryInfo) return null
     320        return e.get("best") ? "best" : null
     321    }
    243322    String getDescription(Object o) {
    244323        def url = getUrl(o)
Note: See TracChangeset for help on using the changeset viewer.