[7726] | 1 | // License: GPL. For details, see LICENSE file.
|
---|
| 2 | /**
|
---|
| 3 | * Compare and analyse the differences of the editor imagery index and the JOSM imagery list.
|
---|
| 4 | * The goal is to keep both lists in sync.
|
---|
| 5 | *
|
---|
| 6 | * The editor imagery index project (https://github.com/osmlab/editor-imagery-index)
|
---|
| 7 | * provides also a version in the JOSM format, but the JSON is the original source
|
---|
| 8 | * format, so we read that.
|
---|
| 9 | *
|
---|
| 10 | * How to run:
|
---|
| 11 | * -----------
|
---|
| 12 | *
|
---|
| 13 | * Main JOSM binary needs to be in classpath, e.g.
|
---|
| 14 | *
|
---|
| 15 | * $ groovy -cp ../dist/josm-custom.jar sync_editor-imagery-index.groovy
|
---|
[9667] | 16 | *
|
---|
[7726] | 17 | * Add option "-h" to show the available command line flags.
|
---|
| 18 | */
|
---|
| 19 | import javax.json.Json
|
---|
| 20 | import javax.json.JsonArray
|
---|
| 21 | import javax.json.JsonObject
|
---|
| 22 | import javax.json.JsonReader
|
---|
| 23 |
|
---|
[9953] | 24 | import org.openstreetmap.josm.data.imagery.ImageryInfo
|
---|
[7726] | 25 | import org.openstreetmap.josm.io.imagery.ImageryReader
|
---|
| 26 |
|
---|
[9880] | 27 | class SyncEditorImageryIndex {
|
---|
[7726] | 28 |
|
---|
| 29 | List<ImageryInfo> josmEntries;
|
---|
| 30 | JsonArray eiiEntries;
|
---|
| 31 |
|
---|
| 32 | def eiiUrls = new HashMap<String, JsonObject>()
|
---|
| 33 | def josmUrls = new HashMap<String, ImageryInfo>()
|
---|
[9667] | 34 |
|
---|
[7726] | 35 | static String eiiInputFile = 'imagery.json'
|
---|
| 36 | static String josmInputFile = 'maps.xml'
|
---|
[11238] | 37 | static String ignoreInputFile = 'maps_ignores.txt'
|
---|
[9505] | 38 | static FileWriter outputFile = null
|
---|
| 39 | static BufferedWriter outputStream = null
|
---|
[10515] | 40 | int skipCount = 0;
|
---|
[11238] | 41 | String skipColor = "greenyellow" // should never be visible
|
---|
[10515] | 42 | def skipEntries = [:]
|
---|
[11238] | 43 | def skipColors = [:]
|
---|
[9505] | 44 |
|
---|
[7726] | 45 | static def options
|
---|
[9658] | 46 |
|
---|
[7726] | 47 | /**
|
---|
| 48 | * Main method.
|
---|
| 49 | */
|
---|
| 50 | static main(def args) {
|
---|
| 51 | parse_command_line_arguments(args)
|
---|
[9880] | 52 | def script = new SyncEditorImageryIndex()
|
---|
[9505] | 53 | script.loadSkip()
|
---|
[9658] | 54 | script.start()
|
---|
[7726] | 55 | script.loadJosmEntries()
|
---|
| 56 | script.loadEIIEntries()
|
---|
| 57 | script.checkInOneButNotTheOther()
|
---|
| 58 | script.checkCommonEntries()
|
---|
[9658] | 59 | script.end()
|
---|
[9505] | 60 | if(outputStream != null) {
|
---|
| 61 | outputStream.close();
|
---|
| 62 | }
|
---|
| 63 | if(outputFile != null) {
|
---|
| 64 | outputFile.close();
|
---|
| 65 | }
|
---|
[7726] | 66 | }
|
---|
[9653] | 67 |
|
---|
[7726] | 68 | /**
|
---|
| 69 | * Parse command line arguments.
|
---|
| 70 | */
|
---|
| 71 | static void parse_command_line_arguments(args) {
|
---|
[9658] | 72 | def cli = new CliBuilder(width: 160)
|
---|
[9505] | 73 | cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
|
---|
| 74 | cli.e(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
|
---|
| 75 | cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
|
---|
[11238] | 76 | cli.i(longOpt:'ignore_input', args:1, argName:"ignore_input", "Input file for the ignore list. Default is $ignoreInputFile (current directory).")
|
---|
[7726] | 77 | cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
|
---|
[9505] | 78 | cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
|
---|
[9658] | 79 | cli.x(longOpt:'xhtmlbody', argName:"xhtmlbody", "create XHTML body for display in a web page")
|
---|
| 80 | cli.X(longOpt:'xhtml', argName:"xhtml", "create XHTML for display in a web page")
|
---|
[9505] | 81 | cli.m(longOpt:'nomissingeii', argName:"nomissingeii", "don't show missing editor imagery index entries")
|
---|
[7726] | 82 | cli.h(longOpt:'help', "show this help")
|
---|
| 83 | options = cli.parse(args)
|
---|
| 84 |
|
---|
| 85 | if (options.h) {
|
---|
| 86 | cli.usage()
|
---|
| 87 | System.exit(0)
|
---|
| 88 | }
|
---|
| 89 | if (options.eii_input) {
|
---|
| 90 | eiiInputFile = options.eii_input
|
---|
| 91 | }
|
---|
| 92 | if (options.josm_input) {
|
---|
| 93 | josmInputFile = options.josm_input
|
---|
| 94 | }
|
---|
[11238] | 95 | if (options.ignore_input) {
|
---|
| 96 | ignoreInputFile = options.ignore_input
|
---|
| 97 | }
|
---|
[9505] | 98 | if (options.output && options.output != "-") {
|
---|
| 99 | outputFile = new FileWriter(options.output)
|
---|
| 100 | outputStream = new BufferedWriter(outputFile)
|
---|
| 101 | }
|
---|
[7726] | 102 | }
|
---|
| 103 |
|
---|
[9505] | 104 | void loadSkip() {
|
---|
[11238] | 105 | FileReader fr = new FileReader(ignoreInputFile)
|
---|
| 106 | def line
|
---|
| 107 |
|
---|
| 108 | while((line = fr.readLine()) != null) {
|
---|
[11242] | 109 | def res = (line =~ /^\|\| *(\d) *\|\| *(EII|Ignore) *\|\| *\{\{\{(.+)\}\}\} *\|\|/)
|
---|
[11238] | 110 | if(res.count)
|
---|
| 111 | {
|
---|
| 112 | skipEntries[res[0][3]] = res[0][1] as int
|
---|
| 113 | if(res[0][2].equals("Ignore")) {
|
---|
| 114 | skipColors[res[0][3]] = "green"
|
---|
| 115 | } else {
|
---|
[11245] | 116 | skipColors[res[0][3]] = "darkgoldenrod"
|
---|
[11238] | 117 | }
|
---|
| 118 | }
|
---|
[11234] | 119 | }
|
---|
[11238] | 120 | }
|
---|
[9653] | 121 |
|
---|
[9658] | 122 | void myprintlnfinal(String s) {
|
---|
| 123 | if(outputStream != null) {
|
---|
| 124 | outputStream.write(s);
|
---|
| 125 | outputStream.newLine();
|
---|
| 126 | } else {
|
---|
| 127 | println s;
|
---|
| 128 | }
|
---|
| 129 | }
|
---|
| 130 |
|
---|
[9505] | 131 | void myprintln(String s) {
|
---|
| 132 | if(skipEntries.containsKey(s)) {
|
---|
| 133 | skipCount = skipEntries.get(s)
|
---|
[9658] | 134 | skipEntries.remove(s)
|
---|
[11238] | 135 | if(skipColors.containsKey(s)) {
|
---|
| 136 | skipColor = skipColors.get(s)
|
---|
| 137 | } else {
|
---|
| 138 | skipColor = "greenyellow"
|
---|
| 139 | }
|
---|
[9505] | 140 | }
|
---|
| 141 | if(skipCount) {
|
---|
| 142 | skipCount -= 1;
|
---|
[9658] | 143 | if(options.xhtmlbody || options.xhtml) {
|
---|
[11238] | 144 | s = "<pre style=\"margin:3px;color:"+skipColor+"\">"+s.replaceAll("&","&").replaceAll("<","<").replaceAll(">",">")+"</pre>"
|
---|
[9658] | 145 | }
|
---|
[9662] | 146 | if (!options.noskip) {
|
---|
| 147 | return;
|
---|
| 148 | }
|
---|
[9658] | 149 | } else if(options.xhtmlbody || options.xhtml) {
|
---|
| 150 | String color = s.startsWith("***") ? "black" : (s.startsWith("+ ") ? "blue" : "red")
|
---|
| 151 | s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&","&").replaceAll("<","<").replaceAll(">",">")+"</pre>"
|
---|
[9505] | 152 | }
|
---|
[9658] | 153 | myprintlnfinal(s)
|
---|
| 154 | }
|
---|
| 155 |
|
---|
| 156 | void start() {
|
---|
| 157 | if (options.xhtml) {
|
---|
| 158 | myprintlnfinal "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
|
---|
| 159 | myprintlnfinal "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/><title>JOSM - EII differences</title></head><body>\n"
|
---|
[9505] | 160 | }
|
---|
| 161 | }
|
---|
[9653] | 162 |
|
---|
[9658] | 163 | void end() {
|
---|
| 164 | for (def s: skipEntries.keySet()) {
|
---|
| 165 | myprintln "+++ Obsolete skip entry: " + s
|
---|
| 166 | }
|
---|
| 167 | if (options.xhtml) {
|
---|
| 168 | myprintlnfinal "</body></html>\n"
|
---|
| 169 | }
|
---|
| 170 | }
|
---|
| 171 |
|
---|
[7726] | 172 | void loadEIIEntries() {
|
---|
| 173 | FileReader fr = new FileReader(eiiInputFile)
|
---|
| 174 | JsonReader jr = Json.createReader(fr)
|
---|
| 175 | eiiEntries = jr.readArray()
|
---|
| 176 | jr.close()
|
---|
[9653] | 177 |
|
---|
[7726] | 178 | for (def e : eiiEntries) {
|
---|
| 179 | def url = getUrl(e)
|
---|
[9653] | 180 | if (url.contains("{z}")) {
|
---|
| 181 | myprintln "+++ EII-URL uses {z} instead of {zoom}: "+url
|
---|
| 182 | url = url.replace("{z}","{zoom}")
|
---|
| 183 | }
|
---|
[9505] | 184 | if (eiiUrls.containsKey(url)) {
|
---|
| 185 | myprintln "+++ EII-URL is not unique: "+url
|
---|
| 186 | } else {
|
---|
| 187 | eiiUrls.put(url, e)
|
---|
| 188 | }
|
---|
[7726] | 189 | }
|
---|
[9505] | 190 | myprintln "*** Loaded ${eiiEntries.size()} entries (EII). ***"
|
---|
[7726] | 191 | }
|
---|
| 192 |
|
---|
| 193 | void loadJosmEntries() {
|
---|
| 194 | def reader = new ImageryReader(josmInputFile)
|
---|
| 195 | josmEntries = reader.parse()
|
---|
[9667] | 196 |
|
---|
[7726] | 197 | for (def e : josmEntries) {
|
---|
| 198 | def url = getUrl(e)
|
---|
[9658] | 199 | if (url.contains("{z}")) {
|
---|
| 200 | myprintln "+++ JOSM-URL uses {z} instead of {zoom}: "+url
|
---|
| 201 | url = url.replace("{z}","{zoom}")
|
---|
| 202 | }
|
---|
[9505] | 203 | if (josmUrls.containsKey(url)) {
|
---|
| 204 | myprintln "+++ JOSM-URL is not unique: "+url
|
---|
| 205 | } else {
|
---|
| 206 | josmUrls.put(url, e)
|
---|
[7726] | 207 | }
|
---|
[9658] | 208 | for (def m : e.getMirrors()) {
|
---|
| 209 | url = getUrl(m)
|
---|
| 210 | if (josmUrls.containsKey(url)) {
|
---|
| 211 | myprintln "+++ JOSM-Mirror-URL is not unique: "+url
|
---|
| 212 | } else {
|
---|
| 213 | josmUrls.put(url, m)
|
---|
| 214 | }
|
---|
| 215 | }
|
---|
[7726] | 216 | }
|
---|
[9505] | 217 | myprintln "*** Loaded ${josmEntries.size()} entries (JOSM). ***"
|
---|
[7726] | 218 | }
|
---|
| 219 |
|
---|
| 220 | List inOneButNotTheOther(Map m1, Map m2) {
|
---|
| 221 | def l = []
|
---|
| 222 | for (def url : m1.keySet()) {
|
---|
| 223 | if (!m2.containsKey(url)) {
|
---|
| 224 | def name = getName(m1.get(url))
|
---|
| 225 | l += " "+getDescription(m1.get(url))
|
---|
| 226 | }
|
---|
| 227 | }
|
---|
| 228 | l.sort()
|
---|
| 229 | }
|
---|
[9667] | 230 |
|
---|
[7726] | 231 | void checkInOneButNotTheOther() {
|
---|
| 232 | def l1 = inOneButNotTheOther(eiiUrls, josmUrls)
|
---|
[9505] | 233 | myprintln "*** URLs found in EII but not in JOSM (${l1.size()}): ***"
|
---|
[9658] | 234 | if (!l1.isEmpty()) {
|
---|
[9516] | 235 | for (def l : l1)
|
---|
[9658] | 236 | myprintln "-"+l
|
---|
[7726] | 237 | }
|
---|
| 238 |
|
---|
[9505] | 239 | if (options.nomissingeii)
|
---|
| 240 | return
|
---|
[7726] | 241 | def l2 = inOneButNotTheOther(josmUrls, eiiUrls)
|
---|
[9505] | 242 | myprintln "*** URLs found in JOSM but not in EII (${l2.size()}): ***"
|
---|
[9658] | 243 | if (!l2.isEmpty()) {
|
---|
[9516] | 244 | for (def l : l2)
|
---|
[9658] | 245 | myprintln "+" + l
|
---|
[7726] | 246 | }
|
---|
| 247 | }
|
---|
[9667] | 248 |
|
---|
[7726] | 249 | void checkCommonEntries() {
|
---|
[9505] | 250 | myprintln "*** Same URL, but different name: ***"
|
---|
[7726] | 251 | for (def url : eiiUrls.keySet()) {
|
---|
| 252 | def e = eiiUrls.get(url)
|
---|
| 253 | if (!josmUrls.containsKey(url)) continue
|
---|
| 254 | def j = josmUrls.get(url)
|
---|
| 255 | if (!getName(e).equals(getName(j))) {
|
---|
[9505] | 256 | myprintln " name differs: $url"
|
---|
[11246] | 257 | myprintln " (EII): ${getName(e)}"
|
---|
[9505] | 258 | myprintln " (JOSM): ${getName(j)}"
|
---|
[7726] | 259 | }
|
---|
| 260 | }
|
---|
[9667] | 261 |
|
---|
[9505] | 262 | myprintln "*** Same URL, but different type: ***"
|
---|
[7726] | 263 | for (def url : eiiUrls.keySet()) {
|
---|
| 264 | def e = eiiUrls.get(url)
|
---|
| 265 | if (!josmUrls.containsKey(url)) continue
|
---|
| 266 | def j = josmUrls.get(url)
|
---|
| 267 | if (!getType(e).equals(getType(j))) {
|
---|
[9505] | 268 | myprintln " type differs: ${getName(j)} - $url"
|
---|
[11246] | 269 | myprintln " (EII): ${getType(e)}"
|
---|
[9505] | 270 | myprintln " (JOSM): ${getType(j)}"
|
---|
[7726] | 271 | }
|
---|
| 272 | }
|
---|
[9667] | 273 |
|
---|
[9505] | 274 | myprintln "*** Same URL, but different zoom bounds: ***"
|
---|
[7726] | 275 | for (def url : eiiUrls.keySet()) {
|
---|
| 276 | def e = eiiUrls.get(url)
|
---|
| 277 | if (!josmUrls.containsKey(url)) continue
|
---|
| 278 | def j = josmUrls.get(url)
|
---|
| 279 |
|
---|
| 280 | Integer eMinZoom = getMinZoom(e)
|
---|
| 281 | Integer jMinZoom = getMinZoom(j)
|
---|
[9518] | 282 | if (eMinZoom != jMinZoom && !(eMinZoom == 0 && jMinZoom == null)) {
|
---|
[9505] | 283 | myprintln " minzoom differs: ${getDescription(j)}"
|
---|
[11246] | 284 | myprintln " (EII): ${eMinZoom}"
|
---|
[9505] | 285 | myprintln " (JOSM): ${jMinZoom}"
|
---|
[7726] | 286 | }
|
---|
| 287 | Integer eMaxZoom = getMaxZoom(e)
|
---|
| 288 | Integer jMaxZoom = getMaxZoom(j)
|
---|
| 289 | if (eMaxZoom != jMaxZoom) {
|
---|
[9505] | 290 | myprintln " maxzoom differs: ${getDescription(j)}"
|
---|
[11246] | 291 | myprintln " (EII): ${eMaxZoom}"
|
---|
[9505] | 292 | myprintln " (JOSM): ${jMaxZoom}"
|
---|
[7726] | 293 | }
|
---|
| 294 | }
|
---|
[9667] | 295 |
|
---|
[9505] | 296 | myprintln "*** Same URL, but different country code: ***"
|
---|
[7726] | 297 | for (def url : eiiUrls.keySet()) {
|
---|
| 298 | def e = eiiUrls.get(url)
|
---|
| 299 | if (!josmUrls.containsKey(url)) continue
|
---|
| 300 | def j = josmUrls.get(url)
|
---|
| 301 | if (!getCountryCode(e).equals(getCountryCode(j))) {
|
---|
[9505] | 302 | myprintln " country code differs: ${getDescription(j)}"
|
---|
[11246] | 303 | myprintln " (EII): ${getCountryCode(e)}"
|
---|
[9505] | 304 | myprintln " (JOSM): ${getCountryCode(j)}"
|
---|
[7726] | 305 | }
|
---|
| 306 | }
|
---|
[10077] | 307 | /*myprintln "*** Same URL, but different quality: ***"
|
---|
[9505] | 308 | for (def url : eiiUrls.keySet()) {
|
---|
| 309 | def e = eiiUrls.get(url)
|
---|
[9515] | 310 | if (!josmUrls.containsKey(url)) {
|
---|
| 311 | def q = getQuality(e)
|
---|
| 312 | if("best".equals(q)) {
|
---|
| 313 | myprintln " quality best entry not in JOSM for ${getDescription(e)}"
|
---|
| 314 | }
|
---|
| 315 | continue
|
---|
| 316 | }
|
---|
[9505] | 317 | def j = josmUrls.get(url)
|
---|
| 318 | if (!getQuality(e).equals(getQuality(j))) {
|
---|
| 319 | myprintln " quality differs: ${getDescription(j)}"
|
---|
[11246] | 320 | myprintln " (EII): ${getQuality(e)}"
|
---|
[9505] | 321 | myprintln " (JOSM): ${getQuality(j)}"
|
---|
| 322 | }
|
---|
[10077] | 323 | }*/
|
---|
[7726] | 324 | }
|
---|
[9667] | 325 |
|
---|
[7726] | 326 | /**
|
---|
| 327 | * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
|
---|
| 328 | */
|
---|
| 329 | static String getUrl(Object e) {
|
---|
| 330 | if (e instanceof ImageryInfo) return e.url
|
---|
| 331 | return e.getString("url")
|
---|
| 332 | }
|
---|
| 333 | static String getName(Object e) {
|
---|
[10517] | 334 | if (e instanceof ImageryInfo) return e.getOriginalName()
|
---|
[7726] | 335 | return e.getString("name")
|
---|
| 336 | }
|
---|
| 337 | static String getType(Object e) {
|
---|
| 338 | if (e instanceof ImageryInfo) return e.getImageryType().getTypeString()
|
---|
| 339 | return e.getString("type")
|
---|
| 340 | }
|
---|
| 341 | static Integer getMinZoom(Object e) {
|
---|
| 342 | if (e instanceof ImageryInfo) {
|
---|
| 343 | int mz = e.getMinZoom()
|
---|
| 344 | return mz == 0 ? null : mz
|
---|
| 345 | } else {
|
---|
| 346 | def ext = e.getJsonObject("extent")
|
---|
| 347 | if (ext == null) return null
|
---|
| 348 | def num = ext.getJsonNumber("min_zoom")
|
---|
| 349 | if (num == null) return null
|
---|
| 350 | return num.intValue()
|
---|
| 351 | }
|
---|
| 352 | }
|
---|
| 353 | static Integer getMaxZoom(Object e) {
|
---|
| 354 | if (e instanceof ImageryInfo) {
|
---|
| 355 | int mz = e.getMaxZoom()
|
---|
| 356 | return mz == 0 ? null : mz
|
---|
| 357 | } else {
|
---|
| 358 | def ext = e.getJsonObject("extent")
|
---|
| 359 | if (ext == null) return null
|
---|
| 360 | def num = ext.getJsonNumber("max_zoom")
|
---|
| 361 | if (num == null) return null
|
---|
| 362 | return num.intValue()
|
---|
| 363 | }
|
---|
| 364 | }
|
---|
| 365 | static String getCountryCode(Object e) {
|
---|
| 366 | if (e instanceof ImageryInfo) return "".equals(e.getCountryCode()) ? null : e.getCountryCode()
|
---|
| 367 | return e.getString("country_code", null)
|
---|
| 368 | }
|
---|
[9505] | 369 | static String getQuality(Object e) {
|
---|
| 370 | //if (e instanceof ImageryInfo) return "".equals(e.getQuality()) ? null : e.getQuality()
|
---|
| 371 | if (e instanceof ImageryInfo) return null
|
---|
| 372 | return e.get("best") ? "best" : null
|
---|
| 373 | }
|
---|
[7726] | 374 | String getDescription(Object o) {
|
---|
| 375 | def url = getUrl(o)
|
---|
| 376 | def cc = getCountryCode(o)
|
---|
| 377 | if (cc == null) {
|
---|
| 378 | def j = josmUrls.get(url)
|
---|
| 379 | if (j != null) cc = getCountryCode(j)
|
---|
| 380 | if (cc == null) {
|
---|
| 381 | def e = eiiUrls.get(url)
|
---|
| 382 | if (e != null) cc = getCountryCode(e)
|
---|
| 383 | }
|
---|
| 384 | }
|
---|
| 385 | if (cc == null) {
|
---|
| 386 | cc = ''
|
---|
| 387 | } else {
|
---|
| 388 | cc = "[$cc] "
|
---|
| 389 | }
|
---|
| 390 | def d = cc + getName(o) + " - " + getUrl(o)
|
---|
| 391 | if (options.shorten) {
|
---|
| 392 | def MAXLEN = 140
|
---|
| 393 | if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "..."
|
---|
| 394 | }
|
---|
| 395 | return d
|
---|
| 396 | }
|
---|
| 397 | }
|
---|