source: josm/trunk/scripts/SyncEditorImageryIndex.groovy@ 11246

Last change on this file since 11246 was 11246, checked in by Klumbumbus, 7 years ago

see #12706 - typos
  • Property svn:eol-style set to native
File size: 14.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2/**
3 * Compare and analyse the differences of the editor imagery index and the JOSM imagery list.
4 * The goal is to keep both lists in sync.
5 *
6 * The editor imagery index project (https://github.com/osmlab/editor-imagery-index)
7 * provides also a version in the JOSM format, but the JSON is the original source
8 * format, so we read that.
9 *
10 * How to run:
11 * -----------
12 *
13 * Main JOSM binary needs to be in classpath, e.g.
14 *
15 * $ groovy -cp ../dist/josm-custom.jar sync_editor-imagery-index.groovy
16 *
17 * Add option "-h" to show the available command line flags.
18 */
19import javax.json.Json
20import javax.json.JsonArray
21import javax.json.JsonObject
22import javax.json.JsonReader
23
24import org.openstreetmap.josm.data.imagery.ImageryInfo
25import org.openstreetmap.josm.io.imagery.ImageryReader
26
27class SyncEditorImageryIndex {
28
29 List<ImageryInfo> josmEntries;
30 JsonArray eiiEntries;
31
32 def eiiUrls = new HashMap<String, JsonObject>()
33 def josmUrls = new HashMap<String, ImageryInfo>()
34
35 static String eiiInputFile = 'imagery.json'
36 static String josmInputFile = 'maps.xml'
37 static String ignoreInputFile = 'maps_ignores.txt'
38 static FileWriter outputFile = null
39 static BufferedWriter outputStream = null
40 int skipCount = 0;
41 String skipColor = "greenyellow" // should never be visible
42 def skipEntries = [:]
43 def skipColors = [:]
44
45 static def options
46
47 /**
48 * Main method.
49 */
50 static main(def args) {
51 parse_command_line_arguments(args)
52 def script = new SyncEditorImageryIndex()
53 script.loadSkip()
54 script.start()
55 script.loadJosmEntries()
56 script.loadEIIEntries()
57 script.checkInOneButNotTheOther()
58 script.checkCommonEntries()
59 script.end()
60 if(outputStream != null) {
61 outputStream.close();
62 }
63 if(outputFile != null) {
64 outputFile.close();
65 }
66 }
67
68 /**
69 * Parse command line arguments.
70 */
71 static void parse_command_line_arguments(args) {
72 def cli = new CliBuilder(width: 160)
73 cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
74 cli.e(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
75 cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
76 cli.i(longOpt:'ignore_input', args:1, argName:"ignore_input", "Input file for the ignore list. Default is $ignoreInputFile (current directory).")
77 cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
78 cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
79 cli.x(longOpt:'xhtmlbody', argName:"xhtmlbody", "create XHTML body for display in a web page")
80 cli.X(longOpt:'xhtml', argName:"xhtml", "create XHTML for display in a web page")
81 cli.m(longOpt:'nomissingeii', argName:"nomissingeii", "don't show missing editor imagery index entries")
82 cli.h(longOpt:'help', "show this help")
83 options = cli.parse(args)
84
85 if (options.h) {
86 cli.usage()
87 System.exit(0)
88 }
89 if (options.eii_input) {
90 eiiInputFile = options.eii_input
91 }
92 if (options.josm_input) {
93 josmInputFile = options.josm_input
94 }
95 if (options.ignore_input) {
96 ignoreInputFile = options.ignore_input
97 }
98 if (options.output && options.output != "-") {
99 outputFile = new FileWriter(options.output)
100 outputStream = new BufferedWriter(outputFile)
101 }
102 }
103
104 void loadSkip() {
105 FileReader fr = new FileReader(ignoreInputFile)
106 def line
107
108 while((line = fr.readLine()) != null) {
109 def res = (line =~ /^\|\| *(\d) *\|\| *(EII|Ignore) *\|\| *\{\{\{(.+)\}\}\} *\|\|/)
110 if(res.count)
111 {
112 skipEntries[res[0][3]] = res[0][1] as int
113 if(res[0][2].equals("Ignore")) {
114 skipColors[res[0][3]] = "green"
115 } else {
116 skipColors[res[0][3]] = "darkgoldenrod"
117 }
118 }
119 }
120 }
121
122 void myprintlnfinal(String s) {
123 if(outputStream != null) {
124 outputStream.write(s);
125 outputStream.newLine();
126 } else {
127 println s;
128 }
129 }
130
131 void myprintln(String s) {
132 if(skipEntries.containsKey(s)) {
133 skipCount = skipEntries.get(s)
134 skipEntries.remove(s)
135 if(skipColors.containsKey(s)) {
136 skipColor = skipColors.get(s)
137 } else {
138 skipColor = "greenyellow"
139 }
140 }
141 if(skipCount) {
142 skipCount -= 1;
143 if(options.xhtmlbody || options.xhtml) {
144 s = "<pre style=\"margin:3px;color:"+skipColor+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
145 }
146 if (!options.noskip) {
147 return;
148 }
149 } else if(options.xhtmlbody || options.xhtml) {
150 String color = s.startsWith("***") ? "black" : (s.startsWith("+ ") ? "blue" : "red")
151 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")+"</pre>"
152 }
153 myprintlnfinal(s)
154 }
155
156 void start() {
157 if (options.xhtml) {
158 myprintlnfinal "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n"
159 myprintlnfinal "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/><title>JOSM - EII differences</title></head><body>\n"
160 }
161 }
162
163 void end() {
164 for (def s: skipEntries.keySet()) {
165 myprintln "+++ Obsolete skip entry: " + s
166 }
167 if (options.xhtml) {
168 myprintlnfinal "</body></html>\n"
169 }
170 }
171
172 void loadEIIEntries() {
173 FileReader fr = new FileReader(eiiInputFile)
174 JsonReader jr = Json.createReader(fr)
175 eiiEntries = jr.readArray()
176 jr.close()
177
178 for (def e : eiiEntries) {
179 def url = getUrl(e)
180 if (url.contains("{z}")) {
181 myprintln "+++ EII-URL uses {z} instead of {zoom}: "+url
182 url = url.replace("{z}","{zoom}")
183 }
184 if (eiiUrls.containsKey(url)) {
185 myprintln "+++ EII-URL is not unique: "+url
186 } else {
187 eiiUrls.put(url, e)
188 }
189 }
190 myprintln "*** Loaded ${eiiEntries.size()} entries (EII). ***"
191 }
192
193 void loadJosmEntries() {
194 def reader = new ImageryReader(josmInputFile)
195 josmEntries = reader.parse()
196
197 for (def e : josmEntries) {
198 def url = getUrl(e)
199 if (url.contains("{z}")) {
200 myprintln "+++ JOSM-URL uses {z} instead of {zoom}: "+url
201 url = url.replace("{z}","{zoom}")
202 }
203 if (josmUrls.containsKey(url)) {
204 myprintln "+++ JOSM-URL is not unique: "+url
205 } else {
206 josmUrls.put(url, e)
207 }
208 for (def m : e.getMirrors()) {
209 url = getUrl(m)
210 if (josmUrls.containsKey(url)) {
211 myprintln "+++ JOSM-Mirror-URL is not unique: "+url
212 } else {
213 josmUrls.put(url, m)
214 }
215 }
216 }
217 myprintln "*** Loaded ${josmEntries.size()} entries (JOSM). ***"
218 }
219
220 List inOneButNotTheOther(Map m1, Map m2) {
221 def l = []
222 for (def url : m1.keySet()) {
223 if (!m2.containsKey(url)) {
224 def name = getName(m1.get(url))
225 l += " "+getDescription(m1.get(url))
226 }
227 }
228 l.sort()
229 }
230
231 void checkInOneButNotTheOther() {
232 def l1 = inOneButNotTheOther(eiiUrls, josmUrls)
233 myprintln "*** URLs found in EII but not in JOSM (${l1.size()}): ***"
234 if (!l1.isEmpty()) {
235 for (def l : l1)
236 myprintln "-"+l
237 }
238
239 if (options.nomissingeii)
240 return
241 def l2 = inOneButNotTheOther(josmUrls, eiiUrls)
242 myprintln "*** URLs found in JOSM but not in EII (${l2.size()}): ***"
243 if (!l2.isEmpty()) {
244 for (def l : l2)
245 myprintln "+" + l
246 }
247 }
248
249 void checkCommonEntries() {
250 myprintln "*** Same URL, but different name: ***"
251 for (def url : eiiUrls.keySet()) {
252 def e = eiiUrls.get(url)
253 if (!josmUrls.containsKey(url)) continue
254 def j = josmUrls.get(url)
255 if (!getName(e).equals(getName(j))) {
256 myprintln " name differs: $url"
257 myprintln " (EII): ${getName(e)}"
258 myprintln " (JOSM): ${getName(j)}"
259 }
260 }
261
262 myprintln "*** Same URL, but different type: ***"
263 for (def url : eiiUrls.keySet()) {
264 def e = eiiUrls.get(url)
265 if (!josmUrls.containsKey(url)) continue
266 def j = josmUrls.get(url)
267 if (!getType(e).equals(getType(j))) {
268 myprintln " type differs: ${getName(j)} - $url"
269 myprintln " (EII): ${getType(e)}"
270 myprintln " (JOSM): ${getType(j)}"
271 }
272 }
273
274 myprintln "*** Same URL, but different zoom bounds: ***"
275 for (def url : eiiUrls.keySet()) {
276 def e = eiiUrls.get(url)
277 if (!josmUrls.containsKey(url)) continue
278 def j = josmUrls.get(url)
279
280 Integer eMinZoom = getMinZoom(e)
281 Integer jMinZoom = getMinZoom(j)
282 if (eMinZoom != jMinZoom && !(eMinZoom == 0 && jMinZoom == null)) {
283 myprintln " minzoom differs: ${getDescription(j)}"
284 myprintln " (EII): ${eMinZoom}"
285 myprintln " (JOSM): ${jMinZoom}"
286 }
287 Integer eMaxZoom = getMaxZoom(e)
288 Integer jMaxZoom = getMaxZoom(j)
289 if (eMaxZoom != jMaxZoom) {
290 myprintln " maxzoom differs: ${getDescription(j)}"
291 myprintln " (EII): ${eMaxZoom}"
292 myprintln " (JOSM): ${jMaxZoom}"
293 }
294 }
295
296 myprintln "*** Same URL, but different country code: ***"
297 for (def url : eiiUrls.keySet()) {
298 def e = eiiUrls.get(url)
299 if (!josmUrls.containsKey(url)) continue
300 def j = josmUrls.get(url)
301 if (!getCountryCode(e).equals(getCountryCode(j))) {
302 myprintln " country code differs: ${getDescription(j)}"
303 myprintln " (EII): ${getCountryCode(e)}"
304 myprintln " (JOSM): ${getCountryCode(j)}"
305 }
306 }
307 /*myprintln "*** Same URL, but different quality: ***"
308 for (def url : eiiUrls.keySet()) {
309 def e = eiiUrls.get(url)
310 if (!josmUrls.containsKey(url)) {
311 def q = getQuality(e)
312 if("best".equals(q)) {
313 myprintln " quality best entry not in JOSM for ${getDescription(e)}"
314 }
315 continue
316 }
317 def j = josmUrls.get(url)
318 if (!getQuality(e).equals(getQuality(j))) {
319 myprintln " quality differs: ${getDescription(j)}"
320 myprintln " (EII): ${getQuality(e)}"
321 myprintln " (JOSM): ${getQuality(j)}"
322 }
323 }*/
324 }
325
326 /**
327 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
328 */
329 static String getUrl(Object e) {
330 if (e instanceof ImageryInfo) return e.url
331 return e.getString("url")
332 }
333 static String getName(Object e) {
334 if (e instanceof ImageryInfo) return e.getOriginalName()
335 return e.getString("name")
336 }
337 static String getType(Object e) {
338 if (e instanceof ImageryInfo) return e.getImageryType().getTypeString()
339 return e.getString("type")
340 }
341 static Integer getMinZoom(Object e) {
342 if (e instanceof ImageryInfo) {
343 int mz = e.getMinZoom()
344 return mz == 0 ? null : mz
345 } else {
346 def ext = e.getJsonObject("extent")
347 if (ext == null) return null
348 def num = ext.getJsonNumber("min_zoom")
349 if (num == null) return null
350 return num.intValue()
351 }
352 }
353 static Integer getMaxZoom(Object e) {
354 if (e instanceof ImageryInfo) {
355 int mz = e.getMaxZoom()
356 return mz == 0 ? null : mz
357 } else {
358 def ext = e.getJsonObject("extent")
359 if (ext == null) return null
360 def num = ext.getJsonNumber("max_zoom")
361 if (num == null) return null
362 return num.intValue()
363 }
364 }
365 static String getCountryCode(Object e) {
366 if (e instanceof ImageryInfo) return "".equals(e.getCountryCode()) ? null : e.getCountryCode()
367 return e.getString("country_code", null)
368 }
369 static String getQuality(Object e) {
370 //if (e instanceof ImageryInfo) return "".equals(e.getQuality()) ? null : e.getQuality()
371 if (e instanceof ImageryInfo) return null
372 return e.get("best") ? "best" : null
373 }
374 String getDescription(Object o) {
375 def url = getUrl(o)
376 def cc = getCountryCode(o)
377 if (cc == null) {
378 def j = josmUrls.get(url)
379 if (j != null) cc = getCountryCode(j)
380 if (cc == null) {
381 def e = eiiUrls.get(url)
382 if (e != null) cc = getCountryCode(e)
383 }
384 }
385 if (cc == null) {
386 cc = ''
387 } else {
388 cc = "[$cc] "
389 }
390 def d = cc + getName(o) + " - " + getUrl(o)
391 if (options.shorten) {
392 def MAXLEN = 140
393 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "..."
394 }
395 return d
396 }
397}
Note: See TracBrowser for help on using the repository browser.