source: josm/trunk/scripts/SyncEditorImageryIndex.groovy@ 11411

Last change on this file since 11411 was 11411, checked in by stoecker, 7 years ago

see #12706 - improve shape difference display, ignore shapes equivalent to bounds
  • Property svn:eol-style set to native
File size: 16.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2/**
3 * Compare and analyse the differences of the editor imagery index and the JOSM imagery list.
4 * The goal is to keep both lists in sync.
5 *
6 * The editor imagery index project (https://github.com/osmlab/editor-imagery-index)
7 * provides also a version in the JOSM format, but the JSON is the original source
8 * format, so we read that.
9 *
10 * How to run:
11 * -----------
12 *
13 * Main JOSM binary needs to be in classpath, e.g.
14 *
15 * $ groovy -cp ../dist/josm-custom.jar SyncEditorImageryIndex.groovy
16 *
17 * Add option "-h" to show the available command line flags.
18 */
19import javax.json.Json
20import javax.json.JsonArray
21import javax.json.JsonObject
22import javax.json.JsonReader
23
24import org.openstreetmap.josm.data.imagery.ImageryInfo
25import org.openstreetmap.josm.data.imagery.Shape
26import org.openstreetmap.josm.io.imagery.ImageryReader
27
28class SyncEditorImageryIndex {
29
30 List<ImageryInfo> josmEntries;
31 JsonArray eiiEntries;
32
33 def eiiUrls = new HashMap<String, JsonObject>()
34 def josmUrls = new HashMap<String, ImageryInfo>()
35
36 static String eiiInputFile = 'imagery.json'
37 static String josmInputFile = 'maps.xml'
38 static String ignoreInputFile = 'maps_ignores.txt'
39 static FileWriter outputFile = null
40 static BufferedWriter outputStream = null
41 int skipCount = 0;
42 String skipColor = "greenyellow" // should never be visible
43 def skipEntries = [:]
44 def skipColors = [:]
45
46 static def options
47
48 /**
49 * Main method.
50 */
51 static main(def args) {
52 parse_command_line_arguments(args)
53 def script = new SyncEditorImageryIndex()
54 script.loadSkip()
55 script.start()
56 script.loadJosmEntries()
57 script.loadEIIEntries()
58 script.checkInOneButNotTheOther()
59 script.checkCommonEntries()
60 script.end()
61 if(outputStream != null) {
62 outputStream.close();
63 }
64 if(outputFile != null) {
65 outputFile.close();
66 }
67 }
68
69 /**
70 * Parse command line arguments.
71 */
72 static void parse_command_line_arguments(args) {
73 def cli = new CliBuilder(width: 160)
74 cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
75 cli.e(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
76 cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
77 cli.i(longOpt:'ignore_input', args:1, argName:"ignore_input", "Input file for the ignore list. Default is $ignoreInputFile (current directory).")
78 cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
79 cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
80 cli.x(longOpt:'xhtmlbody', argName:"xhtmlbody", "create XHTML body for display in a web page")
81 cli.X(longOpt:'xhtml', argName:"xhtml", "create XHTML for display in a web page")
82 cli.m(longOpt:'nomissingeii', argName:"nomissingeii", "don't show missing editor imagery index entries")
83 cli.h(longOpt:'help', "show this help")
84 options = cli.parse(args)
85
86 if (options.h) {
87 cli.usage()
88 System.exit(0)
89 }
90 if (options.eii_input) {
91 eiiInputFile = options.eii_input
92 }
93 if (options.josm_input) {
94 josmInputFile = options.josm_input
95 }
96 if (options.ignore_input) {
97 ignoreInputFile = options.ignore_input
98 }
99 if (options.output && options.output != "-") {
100 outputFile = new FileWriter(options.output)
101 outputStream = new BufferedWriter(outputFile)
102 }
103 }
104
105 void loadSkip() {
106 FileReader fr = new FileReader(ignoreInputFile)
107 def line
108
109 while((line = fr.readLine()) != null) {
110 def res = (line =~ /^\|\| *(\d) *\|\| *(EII|Ignore) *\|\| *\{\{\{(.+)\}\}\} *\|\|/)
111 if(res.count)
112 {
113 skipEntries[res[0][3]] = res[0][1] as int
114 if(res[0][2].equals("Ignore")) {
115 skipColors[res[0][3]] = "green"
116 } else {
117 skipColors[res[0][3]] = "darkgoldenrod"
118 }
119 }
120 }
121 }
122
123 void myprintlnfinal(String s) {
124 if(outputStream != null) {
125 outputStream.write(s);
126 outputStream.newLine();
127 } else {
128 println s;
129 }
130 }
131
132 void myprintln(String s) {
133 if(skipEntries.containsKey(s)) {
134 skipCount = skipEntries.get(s)
135 skipEntries.remove(s)
136 if(skipColors.containsKey(s)) {
137 skipColor = skipColors.get(s)
138 } else {
139 skipColor = "greenyellow"
140 }
141 }
142 if(skipCount) {
143 skipCount -= 1;
144 if(options.xhtmlbody || options.xhtml) {
145 s = "<pre style=\"margin:3px;color:"+skipColor+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
146 }
147 if (!options.noskip) {
148 return;
149 }
150 } else if(options.xhtmlbody || options.xhtml) {
151 String color = s.startsWith("***") ? "black" : (s.startsWith("+ ") ? "blue" : "red")
152 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
153 }
154 myprintlnfinal(s)
155 }
156
157 void start() {
158 if (options.xhtml) {
159 myprintlnfinal "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
160 myprintlnfinal "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/><title>JOSM - EII differences</title></head><body>\n"
161 }
162 }
163
164 void end() {
165 for (def s: skipEntries.keySet()) {
166 myprintln "+++ Obsolete skip entry: " + s
167 }
168 if (options.xhtml) {
169 myprintlnfinal "</body></html>\n"
170 }
171 }
172
173 void loadEIIEntries() {
174 FileReader fr = new FileReader(eiiInputFile)
175 JsonReader jr = Json.createReader(fr)
176 eiiEntries = jr.readArray()
177 jr.close()
178
179 for (def e : eiiEntries) {
180 def url = getUrl(e)
181 if (url.contains("{z}")) {
182 myprintln "+++ EII-URL uses {z} instead of {zoom}: "+url
183 url = url.replace("{z}","{zoom}")
184 }
185 if (eiiUrls.containsKey(url)) {
186 myprintln "+++ EII-URL is not unique: "+url
187 } else {
188 eiiUrls.put(url, e)
189 }
190 }
191 myprintln "*** Loaded ${eiiEntries.size()} entries (EII). ***"
192 }
193
194 void loadJosmEntries() {
195 def reader = new ImageryReader(josmInputFile)
196 josmEntries = reader.parse()
197
198 for (def e : josmEntries) {
199 def url = getUrl(e)
200 if (url.contains("{z}")) {
201 myprintln "+++ JOSM-URL uses {z} instead of {zoom}: "+url
202 url = url.replace("{z}","{zoom}")
203 }
204 if (josmUrls.containsKey(url)) {
205 myprintln "+++ JOSM-URL is not unique: "+url
206 } else {
207 josmUrls.put(url, e)
208 }
209 for (def m : e.getMirrors()) {
210 url = getUrl(m)
211 if (josmUrls.containsKey(url)) {
212 myprintln "+++ JOSM-Mirror-URL is not unique: "+url
213 } else {
214 josmUrls.put(url, m)
215 }
216 }
217 }
218 myprintln "*** Loaded ${josmEntries.size()} entries (JOSM). ***"
219 }
220
221 List inOneButNotTheOther(Map m1, Map m2) {
222 def l = []
223 for (def url : m1.keySet()) {
224 if (!m2.containsKey(url)) {
225 def name = getName(m1.get(url))
226 l += " "+getDescription(m1.get(url))
227 }
228 }
229 l.sort()
230 }
231
232 void checkInOneButNotTheOther() {
233 def l1 = inOneButNotTheOther(eiiUrls, josmUrls)
234 myprintln "*** URLs found in EII but not in JOSM (${l1.size()}): ***"
235 if (!l1.isEmpty()) {
236 for (def l : l1)
237 myprintln "-"+l
238 }
239
240 if (options.nomissingeii)
241 return
242 def l2 = inOneButNotTheOther(josmUrls, eiiUrls)
243 myprintln "*** URLs found in JOSM but not in EII (${l2.size()}): ***"
244 if (!l2.isEmpty()) {
245 for (def l : l2)
246 myprintln "+" + l
247 }
248 }
249
250 void checkCommonEntries() {
251 myprintln "*** Same URL, but different name: ***"
252 for (def url : eiiUrls.keySet()) {
253 def e = eiiUrls.get(url)
254 if (!josmUrls.containsKey(url)) continue
255 def j = josmUrls.get(url)
256 if (!getName(e).equals(getName(j))) {
257 myprintln " name differs: $url"
258 myprintln " (EII): ${getName(e)}"
259 myprintln " (JOSM): ${getName(j)}"
260 }
261 }
262
263 myprintln "*** Same URL, but different type: ***"
264 for (def url : eiiUrls.keySet()) {
265 def e = eiiUrls.get(url)
266 if (!josmUrls.containsKey(url)) continue
267 def j = josmUrls.get(url)
268 if (!getType(e).equals(getType(j))) {
269 myprintln " type differs: ${getName(j)} - $url"
270 myprintln " (EII): ${getType(e)}"
271 myprintln " (JOSM): ${getType(j)}"
272 }
273 }
274
275 myprintln "*** Same URL, but different zoom bounds: ***"
276 for (def url : eiiUrls.keySet()) {
277 def e = eiiUrls.get(url)
278 if (!josmUrls.containsKey(url)) continue
279 def j = josmUrls.get(url)
280
281 Integer eMinZoom = getMinZoom(e)
282 Integer jMinZoom = getMinZoom(j)
283 if (eMinZoom != jMinZoom && !(eMinZoom == 0 && jMinZoom == null)) {
284 myprintln " minzoom differs: ${getDescription(j)}"
285 myprintln " (EII): ${eMinZoom}"
286 myprintln " (JOSM): ${jMinZoom}"
287 }
288 Integer eMaxZoom = getMaxZoom(e)
289 Integer jMaxZoom = getMaxZoom(j)
290 if (eMaxZoom != jMaxZoom) {
291 myprintln " maxzoom differs: ${getDescription(j)}"
292 myprintln " (EII): ${eMaxZoom}"
293 myprintln " (JOSM): ${jMaxZoom}"
294 }
295 }
296
297 myprintln "*** Same URL, but different country code: ***"
298 for (def url : eiiUrls.keySet()) {
299 def e = eiiUrls.get(url)
300 if (!josmUrls.containsKey(url)) continue
301 def j = josmUrls.get(url)
302 if (!getCountryCode(e).equals(getCountryCode(j))) {
303 myprintln " country code differs: ${getDescription(j)}"
304 myprintln " (EII): ${getCountryCode(e)}"
305 myprintln " (JOSM): ${getCountryCode(j)}"
306 }
307 }
308 /*myprintln "*** Same URL, but different quality: ***"
309 for (def url : eiiUrls.keySet()) {
310 def e = eiiUrls.get(url)
311 if (!josmUrls.containsKey(url)) {
312 def q = getQuality(e)
313 if("best".equals(q)) {
314 myprintln " quality best entry not in JOSM for ${getDescription(e)}"
315 }
316 continue
317 }
318 def j = josmUrls.get(url)
319 if (!getQuality(e).equals(getQuality(j))) {
320 myprintln " quality differs: ${getDescription(j)}"
321 myprintln " (EII): ${getQuality(e)}"
322 myprintln " (JOSM): ${getQuality(j)}"
323 }
324 }*/
325 myprintln "*** Mismatching shapes: ***"
326 for (def url : josmUrls.keySet()) {
327 def j = josmUrls.get(url)
328 def num = 1
329 for (def shape : getShapes(j)) {
330 def p = shape.getPoints()
331 if(!p[0].equals(p[p.size()-1])) {
332 myprintln "+++ JOSM shape $num unclosed: ${getDescription(j)}"
333 }
334 ++num
335 }
336 }
337 for (def url : eiiUrls.keySet()) {
338 def e = eiiUrls.get(url)
339 def num = 1
340 def s = getShapes(e)
341 for (def shape : s) {
342 def p = shape.getPoints()
343 if(!p[0].equals(p[p.size()-1])) {
344 myprintln "+++ EII shape $num unclosed: ${getDescription(e)}"
345 }
346 ++num
347 }
348 if (!josmUrls.containsKey(url)) {
349 continue
350 }
351 def j = josmUrls.get(url)
352 def js = getShapes(j)
353 if(!s.equals(js)) {
354 if(!s.size()) {
355 myprintln " No EII shape: ${getDescription(j)}"
356 } else if(!js.size()) {
357 myprintln " No JOSM shape: ${getDescription(j)}"
358 } else {
359 myprintln " Different shapes: ${getDescription(j)}"
360 }
361 }
362 }
363 }
364
365 /**
366 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
367 */
368 static String getUrl(Object e) {
369 if (e instanceof ImageryInfo) return e.url
370 return e.getString("url")
371 }
372 static String getName(Object e) {
373 if (e instanceof ImageryInfo) return e.getOriginalName()
374 return e.getString("name")
375 }
376 static List<Shape> getShapes(Object e) {
377 if (e instanceof ImageryInfo) {
378 def bounds = e.getBounds();
379 if(bounds != null) {
380 return bounds.getShapes();
381 }
382 return []
383 }
384 def ex = e.get("extent")
385 if(ex != null) {
386 def poly = ex.get("polygon")
387 if(poly != null) {
388 List<Shape> l = []
389 for(def shapes: poly) {
390 def s = new Shape()
391 for(def point: shapes) {
392 def lon = point[0].toString()
393 def lat = point[1].toString()
394 s.addPoint(lat, lon)
395 }
396 l.add(s)
397 }
398 if (l.size() == 1 && l[0].getPoints().size() == 5) {
399 return [] // ignore a bounds equivalent shape
400 }
401 return l
402 }
403 }
404 return []
405 }
406 static String getType(Object e) {
407 if (e instanceof ImageryInfo) return e.getImageryType().getTypeString()
408 return e.getString("type")
409 }
410 static Integer getMinZoom(Object e) {
411 if (e instanceof ImageryInfo) {
412 int mz = e.getMinZoom()
413 return mz == 0 ? null : mz
414 } else {
415 def ext = e.getJsonObject("extent")
416 if (ext == null) return null
417 def num = ext.getJsonNumber("min_zoom")
418 if (num == null) return null
419 return num.intValue()
420 }
421 }
422 static Integer getMaxZoom(Object e) {
423 if (e instanceof ImageryInfo) {
424 int mz = e.getMaxZoom()
425 return mz == 0 ? null : mz
426 } else {
427 def ext = e.getJsonObject("extent")
428 if (ext == null) return null
429 def num = ext.getJsonNumber("max_zoom")
430 if (num == null) return null
431 return num.intValue()
432 }
433 }
434 static String getCountryCode(Object e) {
435 if (e instanceof ImageryInfo) return "".equals(e.getCountryCode()) ? null : e.getCountryCode()
436 return e.getString("country_code", null)
437 }
438 static String getQuality(Object e) {
439 //if (e instanceof ImageryInfo) return "".equals(e.getQuality()) ? null : e.getQuality()
440 if (e instanceof ImageryInfo) return null
441 return e.get("best") ? "best" : null
442 }
443 String getDescription(Object o) {
444 def url = getUrl(o)
445 def cc = getCountryCode(o)
446 if (cc == null) {
447 def j = josmUrls.get(url)
448 if (j != null) cc = getCountryCode(j)
449 if (cc == null) {
450 def e = eiiUrls.get(url)
451 if (e != null) cc = getCountryCode(e)
452 }
453 }
454 if (cc == null) {
455 cc = ''
456 } else {
457 cc = "[$cc] "
458 }
459 def d = cc + getName(o) + " - " + getUrl(o)
460 if (options.shorten) {
461 def MAXLEN = 140
462 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "..."
463 }
464 return d
465 }
466}
Note: See TracBrowser for help on using the repository browser.