source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 18382

Last change on this file since 18382 was 18382, checked in by stoecker, 2 years ago

separate ELI from validity checks
  • Property svn:eol-style set to native
File size: 68.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.BufferedWriter;
8import java.io.IOException;
9import java.io.OutputStreamWriter;
10import java.io.Writer;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.charset.Charset;
15import java.nio.file.Files;
16import java.nio.file.Paths;
17import java.text.DecimalFormat;
18import java.text.ParseException;
19import java.text.SimpleDateFormat;
20import java.util.ArrayList;
21import java.util.Arrays;
22import java.util.Calendar;
23import java.util.Collection;
24import java.util.Collections;
25import java.util.Date;
26import java.util.HashMap;
27import java.util.LinkedList;
28import java.util.List;
29import java.util.Locale;
30import java.util.Map;
31import java.util.Map.Entry;
32import java.util.Objects;
33import java.util.Set;
34import java.util.function.BiConsumer;
35import java.util.function.Function;
36import java.util.regex.Matcher;
37import java.util.regex.Pattern;
38import java.util.stream.Collectors;
39
40import javax.json.Json;
41import javax.json.JsonArray;
42import javax.json.JsonNumber;
43import javax.json.JsonObject;
44import javax.json.JsonReader;
45import javax.json.JsonString;
46import javax.json.JsonValue;
47
48import org.openstreetmap.gui.jmapviewer.Coordinate;
49import org.openstreetmap.josm.data.Preferences;
50import org.openstreetmap.josm.data.imagery.ImageryInfo;
51import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
52import org.openstreetmap.josm.data.imagery.Shape;
53import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
54import org.openstreetmap.josm.data.preferences.JosmUrls;
55import org.openstreetmap.josm.data.projection.Projections;
56import org.openstreetmap.josm.data.sources.SourceInfo;
57import org.openstreetmap.josm.data.validation.routines.DomainValidator;
58import org.openstreetmap.josm.io.imagery.ImageryReader;
59import org.openstreetmap.josm.spi.preferences.Config;
60import org.openstreetmap.josm.tools.ImageProvider;
61import org.openstreetmap.josm.tools.JosmRuntimeException;
62import org.openstreetmap.josm.tools.Logging;
63import org.openstreetmap.josm.tools.OptionParser;
64import org.openstreetmap.josm.tools.OptionParser.OptionCount;
65import org.openstreetmap.josm.tools.ReflectionUtils;
66import org.openstreetmap.josm.tools.Utils;
67import org.xml.sax.SAXException;
68
69/**
70 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
71 * The goal is to keep both lists in sync.
72 *
73 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
74 * provides also a version in the JOSM format, but the GEOJSON is the original source
75 * format, so we read that.
76 *
77 * How to run:
78 * -----------
79 *
80 * Main JOSM binary needs to be in classpath, e.g.
81 *
82 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
83 *
84 * Add option "-h" to show the available command line flags.
85 */
86@SuppressWarnings("unchecked")
87public class SyncEditorLayerIndex {
88
89 private static final int MAXLEN = 140;
90
91 private List<ImageryInfo> josmEntries;
92 private JsonArray eliEntries;
93
94 private final Map<String, JsonObject> eliUrls = new HashMap<>();
95 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
96 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
97 private static final Map<String, String> oldproj = new HashMap<>();
98 private static final List<String> ignoreproj = new LinkedList<>();
99
100 private static String eliInputFile = "imagery_eli.geojson";
101 private static String josmInputFile = "imagery_josm.imagery.xml";
102 private static String ignoreInputFile = "imagery_josm.ignores.txt";
103 private static Writer outputStream;
104 private static String optionOutput;
105 private static boolean optionShorten;
106 private static boolean optionNoSkip;
107 private static boolean optionXhtmlBody;
108 private static boolean optionXhtml;
109 private static String optionEliXml;
110 private static String optionJosmXml;
111 private static String optionEncoding;
112 private static boolean optionNoEli;
113 private Map<String, String> skip = new HashMap<>();
114 private Map<String, String> skipStart = new HashMap<>();
115
116 /**
117 * Main method.
118 * @param args program arguments
119 * @throws IOException if any I/O error occurs
120 * @throws ReflectiveOperationException if any reflective operation error occurs
121 * @throws SAXException if any SAX error occurs
122 */
123 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
124 Locale.setDefault(Locale.ROOT);
125 parseCommandLineArguments(args);
126 Config.setUrlsProvider(JosmUrls.getInstance());
127 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
128 Config.setPreferencesInstance(pref);
129 pref.init(false);
130 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
131 script.setupProj();
132 script.loadSkip();
133 script.start();
134 script.loadJosmEntries();
135 if (optionJosmXml != null) {
136 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionJosmXml), UTF_8)) {
137 script.printentries(script.josmEntries, writer);
138 }
139 }
140 script.loadELIEntries();
141 if (optionEliXml != null) {
142 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionEliXml), UTF_8)) {
143 script.printentries(script.eliEntries, writer);
144 }
145 }
146 script.checkInOneButNotTheOther();
147 script.checkCommonEntries();
148 script.end();
149 if (outputStream != null) {
150 outputStream.close();
151 }
152 }
153
154 /**
155 * Displays help on the console
156 */
157 private static void showHelp() {
158 System.out.println(getHelp());
159 System.exit(0);
160 }
161
162 static String getHelp() {
163 return "usage: java -cp build SyncEditorLayerIndex\n" +
164 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
165 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
166 "Default is imagery_eli.geojson (current directory).\n" +
167 "-h,--help show this help\n" +
168 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
169 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
170 "Default is imagery_josm.imagery.xml (current directory).\n" +
171 "-m,--noeli don't show output for ELI problems\n" +
172 "-n,--noskip don't skip known entries\n" +
173 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
174 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
175 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
176 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
177 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
178 "-X,--xhtml create XHTML for display in a web page\n";
179 }
180
181 /**
182 * Parse command line arguments.
183 * @param args program arguments
184 * @throws IOException in case of I/O error
185 */
186 static void parseCommandLineArguments(String[] args) throws IOException {
187 new OptionParser("JOSM/ELI synchronization script")
188 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
189 .addShortAlias("help", "h")
190 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
191 .addShortAlias("output", "o")
192 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
193 .addShortAlias("eli_input", "e")
194 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
195 .addShortAlias("josm_input", "j")
196 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
197 .addShortAlias("ignore_input", "i")
198 .addFlagParameter("shorten", () -> optionShorten = true)
199 .addShortAlias("shorten", "s")
200 .addFlagParameter("noskip", () -> optionNoSkip = true)
201 .addShortAlias("noskip", "n")
202 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
203 .addShortAlias("xhtmlbody", "x")
204 .addFlagParameter("xhtml", () -> optionXhtml = true)
205 .addShortAlias("xhtml", "X")
206 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
207 .addShortAlias("elixml", "p")
208 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
209 .addShortAlias("josmxml", "q")
210 .addFlagParameter("noeli", () -> optionNoEli = true)
211 .addShortAlias("noeli", "m")
212 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
213 .addShortAlias("encoding", "c")
214 .parseOptionsOrExit(Arrays.asList(args));
215
216 if (optionOutput != null && !"-".equals(optionOutput)) {
217 outputStream = Files.newBufferedWriter(Paths.get(optionOutput), optionEncoding != null ? Charset.forName(optionEncoding) : UTF_8);
218 } else if (optionEncoding != null) {
219 outputStream = new OutputStreamWriter(System.out, optionEncoding);
220 }
221 }
222
223 void setupProj() {
224 oldproj.put("EPSG:3359", "EPSG:3404");
225 oldproj.put("EPSG:3785", "EPSG:3857");
226 oldproj.put("EPSG:31297", "EPGS:31287");
227 oldproj.put("EPSG:31464", "EPSG:31468");
228 oldproj.put("EPSG:54004", "EPSG:3857");
229 oldproj.put("EPSG:102100", "EPSG:3857");
230 oldproj.put("EPSG:102113", "EPSG:3857");
231 oldproj.put("EPSG:900913", "EPGS:3857");
232 ignoreproj.add("EPSG:4267");
233 ignoreproj.add("EPSG:5221");
234 ignoreproj.add("EPSG:5514");
235 ignoreproj.add("EPSG:32019");
236 ignoreproj.add("EPSG:102066");
237 ignoreproj.add("EPSG:102067");
238 ignoreproj.add("EPSG:102685");
239 ignoreproj.add("EPSG:102711");
240 }
241
242 void loadSkip() throws IOException {
243 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
244 try (BufferedReader fr = Files.newBufferedReader(Paths.get(ignoreInputFile), UTF_8)) {
245 String line;
246
247 while ((line = fr.readLine()) != null) {
248 Matcher res = pattern.matcher(line);
249 if (res.matches()) {
250 String s = res.group(2);
251 if (s.endsWith("...")) {
252 s = s.substring(0, s.length() - 3);
253 if ("Ignore".equals(res.group(1))) {
254 skipStart.put(s, "green");
255 } else {
256 skipStart.put(s, "darkgoldenrod");
257 }
258 } else {
259 if ("Ignore".equals(res.group(1))) {
260 skip.put(s, "green");
261 } else {
262 skip.put(s, "darkgoldenrod");
263 }
264 }
265 }
266 }
267 }
268 }
269
270 void myprintlnfinal(String s) {
271 if (outputStream != null) {
272 try {
273 outputStream.write(s + System.getProperty("line.separator"));
274 } catch (IOException e) {
275 throw new JosmRuntimeException(e);
276 }
277 } else {
278 System.out.println(s);
279 }
280 }
281
282 String isSkipString(String s) {
283 if (skip.containsKey(s))
284 return skip.get(s);
285 for (Entry<String, String> str : skipStart.entrySet()) {
286 if (s.startsWith(str.getKey())) {
287 skipStart.remove(str.getKey());
288 return str.getValue();
289 }
290 }
291 return null;
292 }
293
294 void myprintln(String s) {
295 String color;
296 if ((color = isSkipString(s)) != null) {
297 skip.remove(s);
298 if (optionXhtmlBody || optionXhtml) {
299 s = "<pre style=\"margin:3px;color:"+color+"\">"
300 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
301 }
302 if (!optionNoSkip) {
303 return;
304 }
305 } else if (optionXhtmlBody || optionXhtml) {
306 color =
307 s.startsWith("***") ? "black" :
308 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
309 (s.startsWith("#") ? "indigo" :
310 (s.startsWith("!") ? "orange" :
311 (s.startsWith("~") ? "red" : "brown"))));
312 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
313 }
314 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
315 return;
316 }
317 myprintlnfinal(s);
318 }
319
320 void start() {
321 if (optionXhtml) {
322 myprintlnfinal(
323 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
324 myprintlnfinal(
325 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
326 "<title>JOSM - ELI differences</title></head><body>\n");
327 }
328 }
329
330 void end() {
331 for (String s : skip.keySet()) {
332 myprintln("+++ Obsolete skip entry: " + s);
333 }
334 for (String s : skipStart.keySet()) {
335 myprintln("+++ Obsolete skip entry: " + s + "...");
336 }
337 if (optionXhtml) {
338 myprintlnfinal("</body></html>\n");
339 }
340 }
341
342 void loadELIEntries() throws IOException {
343 try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(eliInputFile), UTF_8))) {
344 eliEntries = jr.readObject().getJsonArray("features");
345 }
346
347 for (JsonValue e : eliEntries) {
348 String url = getUrlStripped(e);
349 if (url.contains("{z}")) {
350 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+getDescription(e));
351 url = url.replace("{z}", "{zoom}");
352 }
353 if (eliUrls.containsKey(url)) {
354 myprintln("+++ ELI-URL is not unique: "+url);
355 } else {
356 eliUrls.put(url, e.asJsonObject());
357 }
358 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
359 if (s != null) {
360 String urlLc = url.toLowerCase(Locale.ENGLISH);
361 List<String> old = new LinkedList<>();
362 for (JsonValue p : s) {
363 String proj = ((JsonString) p).getString();
364 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
365 old.add(proj);
366 }
367 }
368 if (!old.isEmpty()) {
369 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
370 }
371 }
372 }
373 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
374 }
375
376 String cdata(String s) {
377 return cdata(s, false);
378 }
379
380 String cdata(String s, boolean escape) {
381 if (escape) {
382 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
383 } else if (s.matches(".*[<>&].*"))
384 return "<![CDATA["+s+"]]>";
385 return s;
386 }
387
388 String maininfo(Object entry, String offset) {
389 String t = getType(entry);
390 String res = offset + "<type>"+t+"</type>\n";
391 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
392 if (getMinZoom(entry) != null)
393 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
394 if (getMaxZoom(entry) != null)
395 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
396 if ("wms".equals(t)) {
397 List<String> p = getProjections(entry);
398 if (p != null) {
399 res += offset + "<projections>\n";
400 for (String c : p) {
401 res += offset + " <code>"+c+"</code>\n";
402 }
403 res += offset + "</projections>\n";
404 }
405 }
406 return res;
407 }
408
409 void printentries(List<?> entries, Writer stream) throws IOException {
410 DecimalFormat df = new DecimalFormat("#.#######");
411 df.setRoundingMode(java.math.RoundingMode.CEILING);
412 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
413 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
414 for (Object e : entries) {
415 stream.write(" <entry"
416 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
417 + (getOverlay(e) ? " overlay=\"true\"" : "")
418 + ">\n");
419 String t;
420 if (isNotBlank(t = getName(e)))
421 stream.write(" <name>"+cdata(t, true)+"</name>\n");
422 if (isNotBlank(t = getId(e)))
423 stream.write(" <id>"+t+"</id>\n");
424 if (isNotBlank(t = getCategory(e)))
425 stream.write(" <category>"+t+"</category>\n");
426 if (isNotBlank(t = getDate(e)))
427 stream.write(" <date>"+t+"</date>\n");
428 if (isNotBlank(t = getCountryCode(e)))
429 stream.write(" <country-code>"+t+"</country-code>\n");
430 if ((getDefault(e)))
431 stream.write(" <default>true</default>\n");
432 stream.write(maininfo(e, " "));
433 if (isNotBlank(t = getAttributionText(e)))
434 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
435 if (isNotBlank(t = getAttributionUrl(e)))
436 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
437 if (isNotBlank(t = getLogoImage(e)))
438 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
439 if (isNotBlank(t = getLogoUrl(e)))
440 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
441 if (isNotBlank(t = getTermsOfUseText(e)))
442 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
443 if (isNotBlank(t = getTermsOfUseUrl(e)))
444 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
445 if (isNotBlank(t = getPermissionReferenceUrl(e)))
446 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
447 if (isNotBlank(t = getPrivacyPolicyUrl(e)))
448 stream.write(" <privacy-policy-url>"+cdata(t)+"</privacy-policy-url>\n");
449 if ((getValidGeoreference(e)))
450 stream.write(" <valid-georeference>true</valid-georeference>\n");
451 if (isNotBlank(t = getIcon(e)))
452 stream.write(" <icon>"+cdata(t)+"</icon>\n");
453 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
454 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
455 }
456 for (ImageryInfo m : getMirrors(e)) {
457 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
458 }
459 double minlat = 1000;
460 double minlon = 1000;
461 double maxlat = -1000;
462 double maxlon = -1000;
463 String shapes = "";
464 String sep = "\n ";
465 try {
466 for (Shape s: getShapes(e)) {
467 shapes += " <shape>";
468 int i = 0;
469 for (Coordinate p: s.getPoints()) {
470 double lat = p.getLat();
471 double lon = p.getLon();
472 if (lat > maxlat) maxlat = lat;
473 if (lon > maxlon) maxlon = lon;
474 if (lat < minlat) minlat = lat;
475 if (lon < minlon) minlon = lon;
476 if ((i++ % 3) == 0) {
477 shapes += sep + " ";
478 }
479 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
480 }
481 shapes += sep + "</shape>\n";
482 }
483 } catch (IllegalArgumentException ignored) {
484 Logging.trace(ignored);
485 }
486 if (!shapes.isEmpty()) {
487 stream.write(" <bounds min-lat='"+df.format(minlat)
488 +"' min-lon='"+df.format(minlon)
489 +"' max-lat='"+df.format(maxlat)
490 +"' max-lon='"+df.format(maxlon)+"'>\n");
491 stream.write(shapes + " </bounds>\n");
492 }
493 stream.write(" </entry>\n");
494 }
495 stream.write("</imagery>\n");
496 stream.close();
497 }
498
499 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
500 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
501 josmEntries = reader.parse();
502 }
503
504 for (ImageryInfo e : josmEntries) {
505 if (isBlank(getUrl(e))) {
506 myprintln("~~~ JOSM-Entry without URL: " + getDescription(e));
507 continue;
508 }
509 if (isBlank(e.getDate()) && e.getDate() != null) {
510 myprintln("~~~ JOSM-Entry with empty Date: " + getDescription(e));
511 continue;
512 }
513 if (isBlank(getName(e))) {
514 myprintln("~~~ JOSM-Entry without Name: " + getDescription(e));
515 continue;
516 }
517 String url = getUrlStripped(e);
518 if (url.contains("{z}")) {
519 myprintln("~~~ JOSM-URL uses {z} instead of {zoom}: "+getDescription(e));
520 url = url.replace("{z}", "{zoom}");
521 }
522 if (josmUrls.containsKey(url)) {
523 myprintln("~~~ JOSM-URL is not unique: "+url);
524 } else {
525 josmUrls.put(url, e);
526 }
527 for (ImageryInfo m : e.getMirrors()) {
528 url = getUrlStripped(m);
529 Field origNameField = SourceInfo.class.getDeclaredField("origName");
530 ReflectionUtils.setObjectsAccessible(origNameField);
531 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
532 if (josmUrls.containsKey(url)) {
533 myprintln("~~~ JOSM-Mirror-URL is not unique: "+url);
534 } else {
535 josmUrls.put(url, m);
536 josmMirrors.put(url, m);
537 }
538 }
539 }
540 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
541 }
542
543 // catch reordered arguments, make them uppercase, and switches to WMS version 1.3.0
544 String unifyWMS(String url) {
545 String[] x = url.replaceAll("(?i)VERSION=[0-9.]+", "VERSION=x")
546 .replaceAll("(?i)SRS=", "CRS=")
547 .replaceAll("(?i)BBOX=", "BBOX=")
548 .replaceAll("(?i)FORMAT=", "FORMAT=")
549 .replaceAll("(?i)LAYERS=", "LAYERS=")
550 .replaceAll("(?i)MAP=", "MAP=")
551 .replaceAll("(?i)REQUEST=", "REQUEST=")
552 .replaceAll("(?i)SERVICE=", "SERVICE=")
553 .replaceAll("(?i)STYLES=", "STYLES=")
554 .replaceAll("(?i)TRANSPARENT=FALSE", "TRANSPARENT=FALSE")
555 .replaceAll("(?i)TRANSPARENT=TRUE", "TRANSPARENT=TRUE")
556 .replaceAll("(?i)WIDTH=", "WIDTH=")
557 .replaceAll("(?i)HEIGHT=", "HEIGHT=")
558 .split("\\?");
559 return x[0] +"?" + Arrays.stream(x[1].split("&"))
560 .filter(s -> !s.endsWith("=")) // filter empty params
561 .sorted()
562 .collect(Collectors.joining("&"));
563 }
564
565 void checkInOneButNotTheOther() {
566 List<String> le = new LinkedList<>(eliUrls.keySet());
567 List<String> lj = new LinkedList<>(josmUrls.keySet());
568
569 for (String url : new LinkedList<>(le)) {
570 if (lj.contains(url)) {
571 le.remove(url);
572 lj.remove(url);
573 }
574 }
575
576 if (!le.isEmpty() && !lj.isEmpty()) {
577 List<String> ke = new LinkedList<>(le);
578 for (String urle : ke) {
579 JsonObject e = eliUrls.get(urle);
580 String ide = getId(e);
581 String urlhttps = urle.replace("http:", "https:");
582 if (lj.contains(urlhttps)) {
583 myprintln("+ Missing https: "+getDescription(e));
584 eliUrls.put(urlhttps, eliUrls.get(urle));
585 eliUrls.remove(urle);
586 le.remove(urle);
587 lj.remove(urlhttps);
588 } else if (isNotBlank(ide)) {
589 checkUrlsEquality(ide, e, urle, le, lj);
590 }
591 }
592 }
593
594 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
595 Collections.sort(le);
596 if (!le.isEmpty()) {
597 for (String l : le) {
598 myprintln("- " + getDescription(eliUrls.get(l)));
599 }
600 }
601 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
602 Collections.sort(lj);
603 if (!lj.isEmpty()) {
604 for (String l : lj) {
605 myprintln("+ " + getDescription(josmUrls.get(l)));
606 }
607 }
608 }
609
610 void checkUrlsEquality(String ide, JsonObject e, String urle, List<String> le, List<String> lj) {
611 for (String urlj : new LinkedList<>(lj)) {
612 ImageryInfo j = josmUrls.get(urlj);
613 String idj = getId(j);
614
615 if (checkUrlEquality(ide, "id", idj, e, j, urle, urlj, le, lj)) {
616 return;
617 }
618 Collection<String> old = j.getOldIds();
619 if (old != null) {
620 for (String oidj : old) {
621 if (checkUrlEquality(ide, "oldid", oidj, e, j, urle, urlj, le, lj)) {
622 return;
623 }
624 }
625 }
626 }
627 }
628
629 boolean checkUrlEquality(
630 String ide, String idtype, String idj, JsonObject e, ImageryInfo j, String urle, String urlj, List<String> le, List<String> lj) {
631 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
632 if (getType(j).equals("wms") && unifyWMS(urle).equals(unifyWMS(urlj))) {
633 myprintln("# WMS-URL for "+idtype+" "+idj+" modified: "+getDescription(j));
634 } else {
635 myprintln("* URL for "+idtype+" "+idj+" differs ("+urle+"): "+getDescription(j));
636 }
637 le.remove(urle);
638 lj.remove(urlj);
639 // replace key for this entry with JOSM URL
640 eliUrls.remove(urle);
641 eliUrls.put(urlj, e);
642 return true;
643 }
644 return false;
645 }
646
647 void checkCommonEntries() {
648 doSameUrlButDifferentName();
649 doSameUrlButDifferentId();
650 doSameUrlButDifferentType();
651 doSameUrlButDifferentZoomBounds();
652 doSameUrlButDifferentCountryCode();
653 doSameUrlButDifferentQuality();
654 doSameUrlButDifferentDates();
655 doSameUrlButDifferentInformation();
656 doMismatchingShapes();
657 doMismatchingIcons();
658 doMismatchingCategories();
659 doMiscellaneousChecks();
660 }
661
662 void doSameUrlButDifferentName() {
663 myprintln("*** Same URL, but different name: ***");
664 for (String url : eliUrls.keySet()) {
665 JsonObject e = eliUrls.get(url);
666 if (!josmUrls.containsKey(url)) continue;
667 ImageryInfo j = josmUrls.get(url);
668 String ename = getName(e).replace("'", "\u2019");
669 String jname = getName(j).replace("'", "\u2019");
670 if (!ename.equals(jname)) {
671 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
672 }
673 }
674 }
675
676 void doSameUrlButDifferentId() {
677 myprintln("*** Same URL, but different Id: ***");
678 for (String url : eliUrls.keySet()) {
679 JsonObject e = eliUrls.get(url);
680 if (!josmUrls.containsKey(url)) continue;
681 ImageryInfo j = josmUrls.get(url);
682 String ename = getId(e);
683 String jname = getId(j);
684 if (!Objects.equals(ename, jname)) {
685 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
686 }
687 }
688 }
689
690 void doSameUrlButDifferentType() {
691 myprintln("*** Same URL, but different type: ***");
692 for (String url : eliUrls.keySet()) {
693 JsonObject e = eliUrls.get(url);
694 if (!josmUrls.containsKey(url)) continue;
695 ImageryInfo j = josmUrls.get(url);
696 if (!Objects.equals(getType(e), getType(j))) {
697 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
698 }
699 }
700 }
701
702 void doSameUrlButDifferentZoomBounds() {
703 myprintln("*** Same URL, but different zoom bounds: ***");
704 for (String url : eliUrls.keySet()) {
705 JsonObject e = eliUrls.get(url);
706 if (!josmUrls.containsKey(url)) continue;
707 ImageryInfo j = josmUrls.get(url);
708
709 Integer eMinZoom = getMinZoom(e);
710 Integer jMinZoom = getMinZoom(j);
711 /* dont warn for entries copied from the base of the mirror */
712 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
713 jMinZoom = null;
714 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
715 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
716 }
717 Integer eMaxZoom = getMaxZoom(e);
718 Integer jMaxZoom = getMaxZoom(j);
719 /* dont warn for entries copied from the base of the mirror */
720 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
721 jMaxZoom = null;
722 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
723 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
724 }
725 }
726 }
727
728 void doSameUrlButDifferentCountryCode() {
729 myprintln("*** Same URL, but different country code: ***");
730 for (String url : eliUrls.keySet()) {
731 JsonObject e = eliUrls.get(url);
732 if (!josmUrls.containsKey(url)) continue;
733 ImageryInfo j = josmUrls.get(url);
734 String cce = getCountryCode(e);
735 if ("ZZ".equals(cce)) { /* special ELI country code */
736 cce = null;
737 }
738 if (cce != null && !cce.equals(getCountryCode(j))) {
739 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
740 }
741 }
742 }
743
744 void doSameUrlButDifferentQuality() {
745 myprintln("*** Same URL, but different quality: ***");
746 for (String url : eliUrls.keySet()) {
747 JsonObject e = eliUrls.get(url);
748 if (!josmUrls.containsKey(url)) {
749 String q = getQuality(e);
750 if ("eli-best".equals(q)) {
751 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
752 }
753 continue;
754 }
755 ImageryInfo j = josmUrls.get(url);
756 if (!Objects.equals(getQuality(e), getQuality(j))) {
757 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
758 }
759 }
760 }
761
762 void doSameUrlButDifferentDates() {
763 myprintln("*** Same URL, but different dates: ***");
764 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
765 for (String url : eliUrls.keySet()) {
766 String ed = getDate(eliUrls.get(url));
767 if (!josmUrls.containsKey(url)) continue;
768 ImageryInfo j = josmUrls.get(url);
769 String jd = getDate(j);
770 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
771 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
772 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
773 String ed2 = ed;
774 Matcher m = pattern.matcher(ed);
775 if (m.matches()) {
776 Calendar cal = Calendar.getInstance();
777 cal.set(Integer.valueOf(m.group(2)),
778 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
779 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
780 cal.add(Calendar.DAY_OF_MONTH, -1);
781 ed2 = m.group(1) + cal.get(Calendar.YEAR);
782 if (m.group(4) != null)
783 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
784 if (m.group(6) != null)
785 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
786 }
787 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
788 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
789 String t = "'"+ed+"'";
790 if (!ed.equals(ef)) {
791 t += " or '"+ef+"'";
792 }
793 if (jd.isEmpty()) {
794 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
795 } else if (!ed.isEmpty()) {
796 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
797 } else if (!optionNoEli) {
798 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
799 }
800 }
801 }
802 }
803
804 void doSameUrlButDifferentInformation() {
805 myprintln("*** Same URL, but different information: ***");
806 for (String url : eliUrls.keySet()) {
807 if (!josmUrls.containsKey(url)) continue;
808 JsonObject e = eliUrls.get(url);
809 ImageryInfo j = josmUrls.get(url);
810
811 compareDescriptions(e, j);
812 comparePrivacyPolicyUrls(e, j);
813 comparePermissionReferenceUrls(e, j);
814 compareAttributionUrls(e, j);
815 compareAttributionTexts(e, j);
816 compareProjections(e, j);
817 compareDefaults(e, j);
818 compareOverlays(e, j);
819 compareNoTileHeaders(e, j);
820 }
821 }
822
823 void compareDescriptions(JsonObject e, ImageryInfo j) {
824 String et = getDescriptions(e).getOrDefault("en", "");
825 String jt = getDescriptions(j).getOrDefault("en", "");
826 if (!et.equals(jt)) {
827 if (jt.isEmpty()) {
828 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
829 } else if (!et.isEmpty()) {
830 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
831 } else if (!optionNoEli) {
832 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
833 }
834 }
835 }
836
837 void comparePrivacyPolicyUrls(JsonObject e, ImageryInfo j) {
838 String et = getPrivacyPolicyUrl(e);
839 String jt = getPrivacyPolicyUrl(j);
840 if (!Objects.equals(et, jt)) {
841 if (isBlank(jt)) {
842 myprintln("- Missing JOSM privacy policy URL ("+et+"): "+getDescription(j));
843 } else if (isNotBlank(et)) {
844 myprintln("* Privacy policy URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
845 } else if (!optionNoEli) {
846 myprintln("+ Missing ELI privacy policy URL ('"+jt+"'): "+getDescription(j));
847 }
848 }
849 }
850
851 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
852 String et = getPermissionReferenceUrl(e);
853 String jt = getPermissionReferenceUrl(j);
854 String jt2 = getTermsOfUseUrl(j);
855 if (isBlank(jt)) jt = jt2;
856 if (!Objects.equals(et, jt)) {
857 if (isBlank(jt)) {
858 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
859 } else if (isNotBlank(et)) {
860 String ethttps = et.replace("http:", "https:");
861 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
862 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
863 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
864 } else {
865 String ja = getAttributionUrl(j);
866 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
867 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
868 } else {
869 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
870 }
871 }
872 }
873 } else if (!optionNoEli) {
874 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
875 }
876 }
877 }
878
879 void compareAttributionUrls(JsonObject e, ImageryInfo j) {
880 String et = getAttributionUrl(e);
881 String jt = getAttributionUrl(j);
882 if (!Objects.equals(et, jt)) {
883 if (isBlank(jt)) {
884 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
885 } else if (isNotBlank(et)) {
886 String ethttps = et.replace("http:", "https:");
887 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
888 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
889 } else {
890 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
891 }
892 } else if (!optionNoEli) {
893 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
894 }
895 }
896 }
897
898 void compareAttributionTexts(JsonObject e, ImageryInfo j) {
899 String et = getAttributionText(e);
900 String jt = getAttributionText(j);
901 if (!Objects.equals(et, jt)) {
902 if (isBlank(jt)) {
903 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
904 } else if (isNotBlank(et)) {
905 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
906 } else if (!optionNoEli) {
907 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
908 }
909 }
910 }
911
912 void compareProjections(JsonObject e, ImageryInfo j) {
913 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
914 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
915 if (!Objects.equals(et, jt)) {
916 if (isBlank(jt)) {
917 String t = getType(e);
918 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
919 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
920 } else {
921 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
922 }
923 } else if (isNotBlank(et)) {
924 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
925 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
926 } else {
927 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
928 }
929 } else if (!optionNoEli && !"tms".equals(getType(e))) {
930 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
931 }
932 }
933 }
934
935 void compareDefaults(JsonObject e, ImageryInfo j) {
936 boolean ed = getDefault(e);
937 boolean jd = getDefault(j);
938 if (ed != jd) {
939 if (!jd) {
940 myprintln("- Missing JOSM default: "+getDescription(j));
941 } else if (!optionNoEli) {
942 myprintln("+ Missing ELI default: "+getDescription(j));
943 }
944 }
945 }
946
947 void compareOverlays(JsonObject e, ImageryInfo j) {
948 boolean eo = getOverlay(e);
949 boolean jo = getOverlay(j);
950 if (eo != jo) {
951 if (!jo) {
952 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
953 } else if (!optionNoEli) {
954 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
955 }
956 }
957 }
958
959 void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
960 Map<String, Set<String>> eh = getNoTileHeader(e);
961 Map<String, Set<String>> jh = getNoTileHeader(j);
962 if (!Objects.equals(eh, jh)) {
963 if (Utils.isEmpty(jh)) {
964 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
965 } else if (!Utils.isEmpty(eh)) {
966 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
967 } else if (!optionNoEli) {
968 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
969 }
970 }
971 }
972
973 void doMismatchingShapes() {
974 myprintln("*** Mismatching shapes: ***");
975 for (String url : josmUrls.keySet()) {
976 ImageryInfo j = josmUrls.get(url);
977 int num = 1;
978 for (Shape shape : getShapes(j)) {
979 List<Coordinate> p = shape.getPoints();
980 if (!p.get(0).equals(p.get(p.size()-1))) {
981 myprintln("~~~ JOSM shape "+num+" unclosed: "+getDescription(j));
982 }
983 for (int nump = 1; nump < p.size(); ++nump) {
984 if (Objects.equals(p.get(nump-1), p.get(nump))) {
985 myprintln("~~~ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
986 }
987 }
988 ++num;
989 }
990 }
991 for (String url : eliUrls.keySet()) {
992 JsonObject e = eliUrls.get(url);
993 int num = 1;
994 List<Shape> s = null;
995 try {
996 s = getShapes(e);
997 for (Shape shape : s) {
998 List<Coordinate> p = shape.getPoints();
999 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
1000 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
1001 }
1002 for (int nump = 1; nump < p.size(); ++nump) {
1003 if (Objects.equals(p.get(nump-1), p.get(nump))) {
1004 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
1005 }
1006 }
1007 ++num;
1008 }
1009 } catch (IllegalArgumentException err) {
1010 String desc = getDescription(e);
1011 myprintln("+++ ELI shape contains invalid data for "+desc+": "+err.getMessage());
1012 }
1013 if (s == null || !josmUrls.containsKey(url)) {
1014 continue;
1015 }
1016 ImageryInfo j = josmUrls.get(url);
1017 List<Shape> js = getShapes(j);
1018 if (s.isEmpty() && !js.isEmpty()) {
1019 if (!optionNoEli) {
1020 myprintln("+ No ELI shape: "+getDescription(j));
1021 }
1022 } else if (js.isEmpty() && !s.isEmpty()) {
1023 // don't report boundary like 5 point shapes as difference
1024 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
1025 myprintln("- No JOSM shape: "+getDescription(j));
1026 }
1027 } else if (s.size() != js.size()) {
1028 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
1029 } else {
1030 boolean[] edone = new boolean[s.size()];
1031 boolean[] jdone = new boolean[js.size()];
1032 for (int enums = 0; enums < s.size(); ++enums) {
1033 List<Coordinate> ep = s.get(enums).getPoints();
1034 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1035 List<Coordinate> jp = js.get(jnums).getPoints();
1036 if (ep.size() == jp.size() && !jdone[jnums]) {
1037 boolean err = false;
1038 for (int nump = 0; nump < ep.size() && !err; ++nump) {
1039 Coordinate ept = ep.get(nump);
1040 Coordinate jpt = jp.get(nump);
1041 if (differentCoordinate(ept.getLat(), jpt.getLat()) || differentCoordinate(ept.getLon(), jpt.getLon()))
1042 err = true;
1043 }
1044 if (!err) {
1045 edone[enums] = true;
1046 jdone[jnums] = true;
1047 break;
1048 }
1049 }
1050 }
1051 }
1052 for (int enums = 0; enums < s.size(); ++enums) {
1053 List<Coordinate> ep = s.get(enums).getPoints();
1054 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1055 List<Coordinate> jp = js.get(jnums).getPoints();
1056 if (ep.size() == jp.size() && !jdone[jnums]) {
1057 boolean err = false;
1058 for (int nump = 0; nump < ep.size() && !err; ++nump) {
1059 Coordinate ept = ep.get(nump);
1060 Coordinate jpt = jp.get(nump);
1061 if (differentCoordinate(ept.getLat(), jpt.getLat()) || differentCoordinate(ept.getLon(), jpt.getLon())) {
1062 String numtxt = Integer.toString(enums+1);
1063 if (enums != jnums) {
1064 numtxt += '/' + Integer.toString(jnums+1);
1065 }
1066 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
1067 break;
1068 }
1069 }
1070 edone[enums] = true;
1071 jdone[jnums] = true;
1072 break;
1073 }
1074 }
1075 }
1076 for (int enums = 0; enums < s.size(); ++enums) {
1077 List<Coordinate> ep = s.get(enums).getPoints();
1078 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1079 List<Coordinate> jp = js.get(jnums).getPoints();
1080 if (!jdone[jnums]) {
1081 String numtxt = Integer.toString(enums+1);
1082 if (enums != jnums) {
1083 numtxt += '/' + Integer.toString(jnums+1);
1084 }
1085 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
1086 + getDescription(j));
1087 edone[enums] = true;
1088 jdone[jnums] = true;
1089 break;
1090 }
1091 }
1092 }
1093 }
1094 }
1095 }
1096
1097 private boolean differentCoordinate(double v1, double v2) {
1098 double epsilon = 0.00001;
1099 return Math.abs(v1 - v2) > epsilon;
1100 }
1101
1102 void doMismatchingIcons() {
1103 myprintln("*** Mismatching icons: ***");
1104 doMismatching(this::compareIcons);
1105 }
1106
1107 void doMismatchingCategories() {
1108 myprintln("*** Mismatching categories: ***");
1109 doMismatching(this::compareCategories);
1110 }
1111
1112 void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
1113 for (String url : eliUrls.keySet()) {
1114 if (josmUrls.containsKey(url)) {
1115 comparator.accept(josmUrls.get(url), eliUrls.get(url));
1116 }
1117 }
1118 }
1119
1120 void compareIcons(ImageryInfo j, JsonObject e) {
1121 String ij = getIcon(j);
1122 String ie = getIcon(e);
1123 boolean ijok = isNotBlank(ij);
1124 boolean ieok = isNotBlank(ie);
1125 if (ijok && !ieok) {
1126 if (!optionNoEli) {
1127 myprintln("+ No ELI icon: "+getDescription(j));
1128 }
1129 } else if (!ijok && ieok) {
1130 myprintln("- No JOSM icon: "+getDescription(j));
1131 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1132 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1133 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1134 ij.startsWith("data:"))) {
1135 String iehttps = ie.replace("http:", "https:");
1136 if (ij.equals(iehttps)) {
1137 myprintln("+ Different icons: "+getDescription(j));
1138 } else {
1139 myprintln("* Different icons: "+getDescription(j));
1140 }
1141 }
1142 }
1143
1144 void compareCategories(ImageryInfo j, JsonObject e) {
1145 String cj = getCategory(j);
1146 String ce = getCategory(e);
1147 boolean cjok = isNotBlank(cj);
1148 boolean ceok = isNotBlank(ce);
1149 if (cjok && !ceok) {
1150 if (!optionNoEli) {
1151 myprintln("+ No ELI category: "+getDescription(j));
1152 }
1153 } else if (!cjok && ceok) {
1154 myprintln("- No JOSM category: "+getDescription(j));
1155 } else if (cjok && ceok && !Objects.equals(cj, ce)) {
1156 myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
1157 }
1158 }
1159
1160 void doMiscellaneousChecks() {
1161 myprintln("*** Miscellaneous checks: ***");
1162 Map<String, ImageryInfo> josmIds = new HashMap<>();
1163 Collection<String> all = Projections.getAllProjectionCodes();
1164 DomainValidator dv = DomainValidator.getInstance();
1165 for (String url : josmUrls.keySet()) {
1166 ImageryInfo j = josmUrls.get(url);
1167 String id = getId(j);
1168 if ("wms".equals(getType(j))) {
1169 String urlLc = url.toLowerCase(Locale.ENGLISH);
1170 if (getProjections(j).isEmpty()) {
1171 myprintln("~ WMS without projections: "+getDescription(j));
1172 } else {
1173 List<String> unsupported = new LinkedList<>();
1174 List<String> old = new LinkedList<>();
1175 for (String p : getProjectionsUnstripped(j)) {
1176 if ("CRS:84".equals(p)) {
1177 if (!urlLc.contains("version=1.3")) {
1178 myprintln("~ CRS:84 without WMS 1.3: "+getDescription(j));
1179 }
1180 } else if (oldproj.containsKey(p)) {
1181 old.add(p);
1182 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1183 unsupported.add(p);
1184 }
1185 }
1186 if (!unsupported.isEmpty()) {
1187 myprintln("~ Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1188 }
1189 for (String o : old) {
1190 myprintln("~ Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1191 + getDescription(j));
1192 }
1193 }
1194 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1195 myprintln("~ WMS 1.3 with strange CRS specification: "+getDescription(j));
1196 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1197 myprintln("~ WMS 1.1 with strange SRS specification: "+getDescription(j));
1198 }
1199 }
1200 List<String> urls = new LinkedList<>();
1201 if (!"scanex".equals(getType(j))) {
1202 urls.add(url);
1203 }
1204 String jt = getPermissionReferenceUrl(j);
1205 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1206 urls.add(jt);
1207 jt = getTermsOfUseUrl(j);
1208 if (isNotBlank(jt))
1209 urls.add(jt);
1210 jt = getAttributionUrl(j);
1211 if (isNotBlank(jt))
1212 urls.add(jt);
1213 jt = getIcon(j);
1214 if (isNotBlank(jt)) {
1215 if (!jt.startsWith("data:image/"))
1216 urls.add(jt);
1217 else {
1218 try {
1219 new ImageProvider(jt).get();
1220 } catch (RuntimeException e) {
1221 myprintln("~ Strange Icon: "+getDescription(j));
1222 }
1223 }
1224 }
1225 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1226 for (String u : urls) {
1227 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1228 myprintln("~ Strange URL '"+u+"': "+getDescription(j));
1229 } else {
1230 try {
1231 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1232 String domain = jurl.getHost();
1233 int port = jurl.getPort();
1234 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1235 myprintln("~ Strange Domain '"+domain+"': "+getDescription(j));
1236 else if (80 == port || 443 == port) {
1237 myprintln("~ Useless port '"+port+"': "+getDescription(j));
1238 }
1239 } catch (MalformedURLException e) {
1240 myprintln("~ Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1241 }
1242 }
1243 }
1244
1245 if (josmMirrors.containsKey(url)) {
1246 continue;
1247 }
1248 if (isBlank(id)) {
1249 myprintln("~ No JOSM-ID: "+getDescription(j));
1250 } else if (josmIds.containsKey(id)) {
1251 myprintln("~ JOSM-ID "+id+" not unique: "+getDescription(j));
1252 } else {
1253 josmIds.put(id, j);
1254 }
1255 String d = getDate(j);
1256 if (isNotBlank(d)) {
1257 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1258 Matcher m = patternD.matcher(d);
1259 if (!m.matches()) {
1260 myprintln("~ JOSM-Date '"+d+"' is strange: "+getDescription(j));
1261 } else {
1262 try {
1263 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1264 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1265 if (second.compareTo(first) < 0) {
1266 myprintln("~ JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1267 }
1268 } catch (Exception e) {
1269 myprintln("~ JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1270 }
1271 }
1272 }
1273 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1274 myprintln("~ Attribution link without text: "+getDescription(j));
1275 }
1276 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1277 myprintln("~ Logo link without image: "+getDescription(j));
1278 }
1279 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1280 myprintln("~ Terms of Use text without link: "+getDescription(j));
1281 }
1282 List<Shape> js = getShapes(j);
1283 if (!js.isEmpty()) {
1284 double minlat = 1000;
1285 double minlon = 1000;
1286 double maxlat = -1000;
1287 double maxlon = -1000;
1288 for (Shape s: js) {
1289 for (Coordinate p: s.getPoints()) {
1290 double lat = p.getLat();
1291 double lon = p.getLon();
1292 if (lat > maxlat) maxlat = lat;
1293 if (lon > maxlon) maxlon = lon;
1294 if (lat < minlat) minlat = lat;
1295 if (lon < minlon) minlon = lon;
1296 }
1297 }
1298 ImageryBounds b = j.getBounds();
1299 if (differentCoordinate(b.getMinLat(), minlat)
1300 || differentCoordinate(b.getMinLon(), minlon)
1301 || differentCoordinate(b.getMaxLat(), maxlat)
1302 || differentCoordinate(b.getMaxLon(), maxlon)) {
1303 myprintln("~ Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1304 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1305 + getDescription(j));
1306 }
1307 }
1308 List<String> knownCategories = Arrays.asList(
1309 "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
1310 String cat = getCategory(j);
1311 if (isBlank(cat)) {
1312 myprintln("~ No category: "+getDescription(j));
1313 } else if (!knownCategories.contains(cat)) {
1314 myprintln("~ Strange category "+cat+": "+getDescription(j));
1315 }
1316 }
1317 }
1318
1319 /*
1320 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1321 */
1322
1323 static String getUrl(Object e) {
1324 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1325 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1326 }
1327
1328 static String getUrlStripped(Object e) {
1329 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1330 }
1331
1332 static String getDate(Object e) {
1333 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1334 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1335 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1336 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1337 if (!start.isEmpty() && !end.isEmpty())
1338 return start+";"+end;
1339 else if (!start.isEmpty())
1340 return start+";-";
1341 else if (!end.isEmpty())
1342 return "-;"+end;
1343 return "";
1344 }
1345
1346 static Date verifyDate(String year, String month, String day) throws ParseException {
1347 String date;
1348 if (year == null) {
1349 date = "3000-01-01";
1350 } else {
1351 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1352 }
1353 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1354 df.setLenient(false);
1355 return df.parse(date);
1356 }
1357
1358 static String getId(Object e) {
1359 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1360 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1361 }
1362
1363 static String getName(Object e) {
1364 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1365 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1366 }
1367
1368 static List<ImageryInfo> getMirrors(Object e) {
1369 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1370 return Collections.emptyList();
1371 }
1372
1373 static List<String> getProjections(Object e) {
1374 List<String> r = new ArrayList<>();
1375 List<String> u = getProjectionsUnstripped(e);
1376 if (u != null) {
1377 for (String p : u) {
1378 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1379 r.add(p);
1380 }
1381 }
1382 }
1383 return r;
1384 }
1385
1386 static List<String> getProjectionsUnstripped(Object e) {
1387 List<String> r = null;
1388 if (e instanceof ImageryInfo) {
1389 r = ((ImageryInfo) e).getServerProjections();
1390 } else {
1391 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1392 if (s != null) {
1393 r = new ArrayList<>();
1394 for (JsonValue p : s.asJsonArray()) {
1395 r.add(((JsonString) p).getString());
1396 }
1397 }
1398 }
1399 return r != null ? r : Collections.emptyList();
1400 }
1401
1402 static void addJsonShapes(List<Shape> l, JsonArray a) {
1403 if (a.get(0).asJsonArray().get(0) instanceof JsonArray) {
1404 for (JsonValue sub: a.asJsonArray()) {
1405 addJsonShapes(l, sub.asJsonArray());
1406 }
1407 } else {
1408 Shape s = new Shape();
1409 for (JsonValue point: a.asJsonArray()) {
1410 JsonArray ar = point.asJsonArray();
1411 String lon = ar.getJsonNumber(0).toString();
1412 String lat = ar.getJsonNumber(1).toString();
1413 s.addPoint(lat, lon);
1414 }
1415 l.add(s);
1416 }
1417 }
1418
1419 static List<Shape> getShapes(Object e) {
1420 if (e instanceof ImageryInfo) {
1421 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1422 if (bounds != null) {
1423 return bounds.getShapes();
1424 }
1425 return Collections.emptyList();
1426 }
1427 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1428 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1429 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1430 List<Shape> l = new ArrayList<>();
1431 for (JsonValue shapes: poly) {
1432 addJsonShapes(l, shapes.asJsonArray());
1433 }
1434 return l;
1435 }
1436 return Collections.emptyList();
1437 }
1438
1439 static String getType(Object e) {
1440 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1441 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1442 }
1443
1444 static Integer getMinZoom(Object e) {
1445 if (e instanceof ImageryInfo) {
1446 int mz = ((ImageryInfo) e).getMinZoom();
1447 return mz == 0 ? null : mz;
1448 } else {
1449 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1450 if (num == null) return null;
1451 return num.intValue();
1452 }
1453 }
1454
1455 static Integer getMaxZoom(Object e) {
1456 if (e instanceof ImageryInfo) {
1457 int mz = ((ImageryInfo) e).getMaxZoom();
1458 return mz == 0 ? null : mz;
1459 } else {
1460 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1461 if (num == null) return null;
1462 return num.intValue();
1463 }
1464 }
1465
1466 static String getCountryCode(Object e) {
1467 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1468 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1469 }
1470
1471 static String getQuality(Object e) {
1472 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1473 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1474 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1475 }
1476
1477 static boolean getOverlay(Object e) {
1478 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1479 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1480 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1481 }
1482
1483 static String getIcon(Object e) {
1484 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1485 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1486 }
1487
1488 static String getAttributionText(Object e) {
1489 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1490 try {
1491 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1492 } catch (NullPointerException ex) {
1493 return null;
1494 }
1495 }
1496
1497 static String getAttributionUrl(Object e) {
1498 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1499 try {
1500 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1501 } catch (NullPointerException ex) {
1502 return null;
1503 }
1504 }
1505
1506 static String getTermsOfUseText(Object e) {
1507 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1508 return null;
1509 }
1510
1511 static String getTermsOfUseUrl(Object e) {
1512 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1513 return null;
1514 }
1515
1516 static String getCategory(Object e) {
1517 if (e instanceof ImageryInfo) {
1518 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1519 }
1520 return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
1521 }
1522
1523 static String getLogoImage(Object e) {
1524 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1525 return null;
1526 }
1527
1528 static String getLogoUrl(Object e) {
1529 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1530 return null;
1531 }
1532
1533 static String getPermissionReferenceUrl(Object e) {
1534 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1535 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1536 }
1537
1538 static String getPrivacyPolicyUrl(Object e) {
1539 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPrivacyPolicyURL();
1540 return ((Map<String, JsonObject>) e).get("properties").getString("privacy_policy_url", null);
1541 }
1542
1543 static Map<String, Set<String>> getNoTileHeader(Object e) {
1544 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1545 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1546 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1547 Function.identity(),
1548 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1549 }
1550
1551 static Map<String, String> getDescriptions(Object e) {
1552 Map<String, String> res = new HashMap<>();
1553 if (e instanceof ImageryInfo) {
1554 String a = ((ImageryInfo) e).getDescription();
1555 if (a != null) res.put("en", a);
1556 } else {
1557 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1558 if (a != null) res.put("en", a.replaceAll("''", "'"));
1559 }
1560 return res;
1561 }
1562
1563 static boolean getValidGeoreference(Object e) {
1564 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1565 return false;
1566 }
1567
1568 static boolean getDefault(Object e) {
1569 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1570 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1571 }
1572
1573 String getDescription(Object o) {
1574 String url = getUrl(o);
1575 String cc = getCountryCode(o);
1576 if (cc == null) {
1577 ImageryInfo j = josmUrls.get(url);
1578 if (j != null) cc = getCountryCode(j);
1579 if (cc == null) {
1580 JsonObject e = eliUrls.get(url);
1581 if (e != null) cc = getCountryCode(e);
1582 }
1583 }
1584 if (cc == null) {
1585 cc = "";
1586 } else {
1587 cc = "["+cc+"] ";
1588 }
1589 String name = getName(o);
1590 String id = getId(o);
1591 String d = cc;
1592 if (!Utils.isEmpty(name)) {
1593 d += name;
1594 if (!Utils.isEmpty(id))
1595 d += " ["+id+"]";
1596 } else if (!Utils.isEmpty(url))
1597 d += url;
1598 if (optionShorten) {
1599 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1600 }
1601 return d;
1602 }
1603}
Note: See TracBrowser for help on using the repository browser.