source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 17442

Last change on this file since 17442 was 17332, checked in by Don-vip, 3 years ago

fix #20131 - fix unit tests and codestyle violations
  • Property svn:eol-style set to native
File size: 68.3 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.BufferedWriter;
8import java.io.IOException;
9import java.io.OutputStreamWriter;
10import java.io.Writer;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.charset.Charset;
15import java.nio.file.Files;
16import java.nio.file.Paths;
17import java.text.DecimalFormat;
18import java.text.ParseException;
19import java.text.SimpleDateFormat;
20import java.util.ArrayList;
21import java.util.Arrays;
22import java.util.Calendar;
23import java.util.Collection;
24import java.util.Collections;
25import java.util.Date;
26import java.util.HashMap;
27import java.util.LinkedList;
28import java.util.List;
29import java.util.Locale;
30import java.util.Map;
31import java.util.Map.Entry;
32import java.util.Objects;
33import java.util.Set;
34import java.util.function.BiConsumer;
35import java.util.function.Function;
36import java.util.regex.Matcher;
37import java.util.regex.Pattern;
38import java.util.stream.Collectors;
39
40import javax.json.Json;
41import javax.json.JsonArray;
42import javax.json.JsonNumber;
43import javax.json.JsonObject;
44import javax.json.JsonReader;
45import javax.json.JsonString;
46import javax.json.JsonValue;
47
48import org.openstreetmap.gui.jmapviewer.Coordinate;
49import org.openstreetmap.josm.data.Preferences;
50import org.openstreetmap.josm.data.imagery.ImageryInfo;
51import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
52import org.openstreetmap.josm.data.imagery.Shape;
53import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
54import org.openstreetmap.josm.data.preferences.JosmUrls;
55import org.openstreetmap.josm.data.projection.Projections;
56import org.openstreetmap.josm.data.sources.SourceInfo;
57import org.openstreetmap.josm.data.validation.routines.DomainValidator;
58import org.openstreetmap.josm.io.imagery.ImageryReader;
59import org.openstreetmap.josm.spi.preferences.Config;
60import org.openstreetmap.josm.tools.ImageProvider;
61import org.openstreetmap.josm.tools.JosmRuntimeException;
62import org.openstreetmap.josm.tools.Logging;
63import org.openstreetmap.josm.tools.OptionParser;
64import org.openstreetmap.josm.tools.OptionParser.OptionCount;
65import org.openstreetmap.josm.tools.ReflectionUtils;
66import org.xml.sax.SAXException;
67
68/**
69 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
70 * The goal is to keep both lists in sync.
71 *
72 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
73 * provides also a version in the JOSM format, but the GEOJSON is the original source
74 * format, so we read that.
75 *
76 * How to run:
77 * -----------
78 *
79 * Main JOSM binary needs to be in classpath, e.g.
80 *
81 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
82 *
83 * Add option "-h" to show the available command line flags.
84 */
85@SuppressWarnings("unchecked")
86public class SyncEditorLayerIndex {
87
88 private static final int MAXLEN = 140;
89
90 private List<ImageryInfo> josmEntries;
91 private JsonArray eliEntries;
92
93 private final Map<String, JsonObject> eliUrls = new HashMap<>();
94 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
95 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
96 private static final Map<String, String> oldproj = new HashMap<>();
97 private static final List<String> ignoreproj = new LinkedList<>();
98
99 private static String eliInputFile = "imagery_eli.geojson";
100 private static String josmInputFile = "imagery_josm.imagery.xml";
101 private static String ignoreInputFile = "imagery_josm.ignores.txt";
102 private static Writer outputStream;
103 private static String optionOutput;
104 private static boolean optionShorten;
105 private static boolean optionNoSkip;
106 private static boolean optionXhtmlBody;
107 private static boolean optionXhtml;
108 private static String optionEliXml;
109 private static String optionJosmXml;
110 private static String optionEncoding;
111 private static boolean optionNoEli;
112 private Map<String, String> skip = new HashMap<>();
113 private Map<String, String> skipStart = new HashMap<>();
114
115 /**
116 * Main method.
117 * @param args program arguments
118 * @throws IOException if any I/O error occurs
119 * @throws ReflectiveOperationException if any reflective operation error occurs
120 * @throws SAXException if any SAX error occurs
121 */
122 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
123 Locale.setDefault(Locale.ROOT);
124 parseCommandLineArguments(args);
125 Config.setUrlsProvider(JosmUrls.getInstance());
126 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
127 Config.setPreferencesInstance(pref);
128 pref.init(false);
129 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
130 script.setupProj();
131 script.loadSkip();
132 script.start();
133 script.loadJosmEntries();
134 if (optionJosmXml != null) {
135 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionJosmXml), UTF_8)) {
136 script.printentries(script.josmEntries, writer);
137 }
138 }
139 script.loadELIEntries();
140 if (optionEliXml != null) {
141 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionEliXml), UTF_8)) {
142 script.printentries(script.eliEntries, writer);
143 }
144 }
145 script.checkInOneButNotTheOther();
146 script.checkCommonEntries();
147 script.end();
148 if (outputStream != null) {
149 outputStream.close();
150 }
151 }
152
153 /**
154 * Displays help on the console
155 */
156 private static void showHelp() {
157 System.out.println(getHelp());
158 System.exit(0);
159 }
160
161 static String getHelp() {
162 return "usage: java -cp build SyncEditorLayerIndex\n" +
163 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
164 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
165 "Default is imagery_eli.geojson (current directory).\n" +
166 "-h,--help show this help\n" +
167 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
168 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
169 "Default is imagery_josm.imagery.xml (current directory).\n" +
170 "-m,--noeli don't show output for ELI problems\n" +
171 "-n,--noskip don't skip known entries\n" +
172 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
173 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
174 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
175 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
176 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
177 "-X,--xhtml create XHTML for display in a web page\n";
178 }
179
180 /**
181 * Parse command line arguments.
182 * @param args program arguments
183 * @throws IOException in case of I/O error
184 */
185 static void parseCommandLineArguments(String[] args) throws IOException {
186 new OptionParser("JOSM/ELI synchronization script")
187 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
188 .addShortAlias("help", "h")
189 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
190 .addShortAlias("output", "o")
191 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
192 .addShortAlias("eli_input", "e")
193 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
194 .addShortAlias("josm_input", "j")
195 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
196 .addShortAlias("ignore_input", "i")
197 .addFlagParameter("shorten", () -> optionShorten = true)
198 .addShortAlias("shorten", "s")
199 .addFlagParameter("noskip", () -> optionNoSkip = true)
200 .addShortAlias("noskip", "n")
201 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
202 .addShortAlias("xhtmlbody", "x")
203 .addFlagParameter("xhtml", () -> optionXhtml = true)
204 .addShortAlias("xhtml", "X")
205 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
206 .addShortAlias("elixml", "p")
207 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
208 .addShortAlias("josmxml", "q")
209 .addFlagParameter("noeli", () -> optionNoEli = true)
210 .addShortAlias("noeli", "m")
211 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
212 .addShortAlias("encoding", "c")
213 .parseOptionsOrExit(Arrays.asList(args));
214
215 if (optionOutput != null && !"-".equals(optionOutput)) {
216 outputStream = Files.newBufferedWriter(Paths.get(optionOutput), optionEncoding != null ? Charset.forName(optionEncoding) : UTF_8);
217 } else if (optionEncoding != null) {
218 outputStream = new OutputStreamWriter(System.out, optionEncoding);
219 }
220 }
221
222 void setupProj() {
223 oldproj.put("EPSG:3359", "EPSG:3404");
224 oldproj.put("EPSG:3785", "EPSG:3857");
225 oldproj.put("EPSG:31297", "EPGS:31287");
226 oldproj.put("EPSG:31464", "EPSG:31468");
227 oldproj.put("EPSG:54004", "EPSG:3857");
228 oldproj.put("EPSG:102100", "EPSG:3857");
229 oldproj.put("EPSG:102113", "EPSG:3857");
230 oldproj.put("EPSG:900913", "EPGS:3857");
231 ignoreproj.add("EPSG:4267");
232 ignoreproj.add("EPSG:5221");
233 ignoreproj.add("EPSG:5514");
234 ignoreproj.add("EPSG:32019");
235 ignoreproj.add("EPSG:102066");
236 ignoreproj.add("EPSG:102067");
237 ignoreproj.add("EPSG:102685");
238 ignoreproj.add("EPSG:102711");
239 }
240
241 void loadSkip() throws IOException {
242 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
243 try (BufferedReader fr = Files.newBufferedReader(Paths.get(ignoreInputFile), UTF_8)) {
244 String line;
245
246 while ((line = fr.readLine()) != null) {
247 Matcher res = pattern.matcher(line);
248 if (res.matches()) {
249 String s = res.group(2);
250 if (s.endsWith("...")) {
251 s = s.substring(0, s.length() - 3);
252 if ("Ignore".equals(res.group(1))) {
253 skipStart.put(s, "green");
254 } else {
255 skipStart.put(s, "darkgoldenrod");
256 }
257 } else {
258 if ("Ignore".equals(res.group(1))) {
259 skip.put(s, "green");
260 } else {
261 skip.put(s, "darkgoldenrod");
262 }
263 }
264 }
265 }
266 }
267 }
268
269 void myprintlnfinal(String s) {
270 if (outputStream != null) {
271 try {
272 outputStream.write(s + System.getProperty("line.separator"));
273 } catch (IOException e) {
274 throw new JosmRuntimeException(e);
275 }
276 } else {
277 System.out.println(s);
278 }
279 }
280
281 String isSkipString(String s) {
282 if (skip.containsKey(s))
283 return skip.get(s);
284 for (Entry<String, String> str : skipStart.entrySet()) {
285 if (s.startsWith(str.getKey())) {
286 skipStart.remove(str.getKey());
287 return str.getValue();
288 }
289 }
290 return null;
291 }
292
293 void myprintln(String s) {
294 String color;
295 if ((color = isSkipString(s)) != null) {
296 skip.remove(s);
297 if (optionXhtmlBody || optionXhtml) {
298 s = "<pre style=\"margin:3px;color:"+color+"\">"
299 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
300 }
301 if (!optionNoSkip) {
302 return;
303 }
304 } else if (optionXhtmlBody || optionXhtml) {
305 color =
306 s.startsWith("***") ? "black" :
307 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
308 (s.startsWith("#") ? "indigo" :
309 (s.startsWith("!") ? "orange" : "red")));
310 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
311 }
312 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
313 return;
314 }
315 myprintlnfinal(s);
316 }
317
318 void start() {
319 if (optionXhtml) {
320 myprintlnfinal(
321 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
322 myprintlnfinal(
323 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
324 "<title>JOSM - ELI differences</title></head><body>\n");
325 }
326 }
327
328 void end() {
329 for (String s : skip.keySet()) {
330 myprintln("+++ Obsolete skip entry: " + s);
331 }
332 for (String s : skipStart.keySet()) {
333 myprintln("+++ Obsolete skip entry: " + s + "...");
334 }
335 if (optionXhtml) {
336 myprintlnfinal("</body></html>\n");
337 }
338 }
339
340 void loadELIEntries() throws IOException {
341 try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(eliInputFile), UTF_8))) {
342 eliEntries = jr.readObject().getJsonArray("features");
343 }
344
345 for (JsonValue e : eliEntries) {
346 String url = getUrlStripped(e);
347 if (url.contains("{z}")) {
348 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+getDescription(e));
349 url = url.replace("{z}", "{zoom}");
350 }
351 if (eliUrls.containsKey(url)) {
352 myprintln("+++ ELI-URL is not unique: "+url);
353 } else {
354 eliUrls.put(url, e.asJsonObject());
355 }
356 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
357 if (s != null) {
358 String urlLc = url.toLowerCase(Locale.ENGLISH);
359 List<String> old = new LinkedList<>();
360 for (JsonValue p : s) {
361 String proj = ((JsonString) p).getString();
362 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
363 old.add(proj);
364 }
365 }
366 if (!old.isEmpty()) {
367 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
368 }
369 }
370 }
371 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
372 }
373
374 String cdata(String s) {
375 return cdata(s, false);
376 }
377
378 String cdata(String s, boolean escape) {
379 if (escape) {
380 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
381 } else if (s.matches(".*[<>&].*"))
382 return "<![CDATA["+s+"]]>";
383 return s;
384 }
385
386 String maininfo(Object entry, String offset) {
387 String t = getType(entry);
388 String res = offset + "<type>"+t+"</type>\n";
389 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
390 if (getMinZoom(entry) != null)
391 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
392 if (getMaxZoom(entry) != null)
393 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
394 if ("wms".equals(t)) {
395 List<String> p = getProjections(entry);
396 if (p != null) {
397 res += offset + "<projections>\n";
398 for (String c : p) {
399 res += offset + " <code>"+c+"</code>\n";
400 }
401 res += offset + "</projections>\n";
402 }
403 }
404 return res;
405 }
406
407 void printentries(List<?> entries, Writer stream) throws IOException {
408 DecimalFormat df = new DecimalFormat("#.#######");
409 df.setRoundingMode(java.math.RoundingMode.CEILING);
410 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
411 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
412 for (Object e : entries) {
413 stream.write(" <entry"
414 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
415 + (getOverlay(e) ? " overlay=\"true\"" : "")
416 + ">\n");
417 String t;
418 if (isNotBlank(t = getName(e)))
419 stream.write(" <name>"+cdata(t, true)+"</name>\n");
420 if (isNotBlank(t = getId(e)))
421 stream.write(" <id>"+t+"</id>\n");
422 if (isNotBlank(t = getCategory(e)))
423 stream.write(" <category>"+t+"</category>\n");
424 if (isNotBlank(t = getDate(e)))
425 stream.write(" <date>"+t+"</date>\n");
426 if (isNotBlank(t = getCountryCode(e)))
427 stream.write(" <country-code>"+t+"</country-code>\n");
428 if ((getDefault(e)))
429 stream.write(" <default>true</default>\n");
430 stream.write(maininfo(e, " "));
431 if (isNotBlank(t = getAttributionText(e)))
432 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
433 if (isNotBlank(t = getAttributionUrl(e)))
434 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
435 if (isNotBlank(t = getLogoImage(e)))
436 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
437 if (isNotBlank(t = getLogoUrl(e)))
438 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
439 if (isNotBlank(t = getTermsOfUseText(e)))
440 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
441 if (isNotBlank(t = getTermsOfUseUrl(e)))
442 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
443 if (isNotBlank(t = getPermissionReferenceUrl(e)))
444 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
445 if (isNotBlank(t = getPrivacyPolicyUrl(e)))
446 stream.write(" <privacy-policy-url>"+cdata(t)+"</privacy-policy-url>\n");
447 if ((getValidGeoreference(e)))
448 stream.write(" <valid-georeference>true</valid-georeference>\n");
449 if (isNotBlank(t = getIcon(e)))
450 stream.write(" <icon>"+cdata(t)+"</icon>\n");
451 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
452 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
453 }
454 for (ImageryInfo m : getMirrors(e)) {
455 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
456 }
457 double minlat = 1000;
458 double minlon = 1000;
459 double maxlat = -1000;
460 double maxlon = -1000;
461 String shapes = "";
462 String sep = "\n ";
463 try {
464 for (Shape s: getShapes(e)) {
465 shapes += " <shape>";
466 int i = 0;
467 for (Coordinate p: s.getPoints()) {
468 double lat = p.getLat();
469 double lon = p.getLon();
470 if (lat > maxlat) maxlat = lat;
471 if (lon > maxlon) maxlon = lon;
472 if (lat < minlat) minlat = lat;
473 if (lon < minlon) minlon = lon;
474 if ((i++ % 3) == 0) {
475 shapes += sep + " ";
476 }
477 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
478 }
479 shapes += sep + "</shape>\n";
480 }
481 } catch (IllegalArgumentException ignored) {
482 Logging.trace(ignored);
483 }
484 if (!shapes.isEmpty()) {
485 stream.write(" <bounds min-lat='"+df.format(minlat)
486 +"' min-lon='"+df.format(minlon)
487 +"' max-lat='"+df.format(maxlat)
488 +"' max-lon='"+df.format(maxlon)+"'>\n");
489 stream.write(shapes + " </bounds>\n");
490 }
491 stream.write(" </entry>\n");
492 }
493 stream.write("</imagery>\n");
494 stream.close();
495 }
496
497 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
498 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
499 josmEntries = reader.parse();
500 }
501
502 for (ImageryInfo e : josmEntries) {
503 if (isBlank(getUrl(e))) {
504 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
505 continue;
506 }
507 if (isBlank(e.getDate()) && e.getDate() != null) {
508 myprintln("+++ JOSM-Entry with empty Date: " + getDescription(e));
509 continue;
510 }
511 if (isBlank(getName(e))) {
512 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
513 continue;
514 }
515 String url = getUrlStripped(e);
516 if (url.contains("{z}")) {
517 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+getDescription(e));
518 url = url.replace("{z}", "{zoom}");
519 }
520 if (josmUrls.containsKey(url)) {
521 myprintln("+++ JOSM-URL is not unique: "+url);
522 } else {
523 josmUrls.put(url, e);
524 }
525 for (ImageryInfo m : e.getMirrors()) {
526 url = getUrlStripped(m);
527 Field origNameField = SourceInfo.class.getDeclaredField("origName");
528 ReflectionUtils.setObjectsAccessible(origNameField);
529 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
530 if (josmUrls.containsKey(url)) {
531 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
532 } else {
533 josmUrls.put(url, m);
534 josmMirrors.put(url, m);
535 }
536 }
537 }
538 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
539 }
540
541 // catch reordered arguments and switches to WMS version 1.3.0
542 String unifyWMS(String url) {
543 String[] x = url.replaceAll("(?i)VERSION=[0-9.]+", "VERSION=x").replaceAll("(?i)SRS=", "CRS=").split("\\?");
544 String[] a = x[1].split("&");
545 Arrays.sort(a);
546 url = x[0]+"?"+String.join("&", a);
547 return url;
548 }
549
550 void checkInOneButNotTheOther() {
551 List<String> le = new LinkedList<>(eliUrls.keySet());
552 List<String> lj = new LinkedList<>(josmUrls.keySet());
553
554 List<String> ke = new LinkedList<>(le);
555 for (String url : ke) {
556 if (lj.contains(url)) {
557 le.remove(url);
558 lj.remove(url);
559 }
560 }
561
562 if (!le.isEmpty() && !lj.isEmpty()) {
563 ke = new LinkedList<>(le);
564 for (String urle : ke) {
565 JsonObject e = eliUrls.get(urle);
566 String ide = getId(e);
567 String urlhttps = urle.replace("http:", "https:");
568 if (lj.contains(urlhttps)) {
569 myprintln("+ Missing https: "+getDescription(e));
570 eliUrls.put(urlhttps, eliUrls.get(urle));
571 eliUrls.remove(urle);
572 le.remove(urle);
573 lj.remove(urlhttps);
574 } else if (isNotBlank(ide)) {
575 List<String> kj = new LinkedList<>(lj);
576 for (String urlj : kj) {
577 ImageryInfo j = josmUrls.get(urlj);
578 String idj = getId(j);
579
580 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
581 if (getType(j).equals("wms") && unifyWMS(urle).equals(unifyWMS(urlj))) {
582 myprintln("# WMS-URL for id "+idj+" modified: "+getDescription(j));
583 } else {
584 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
585 }
586 le.remove(urle);
587 lj.remove(urlj);
588 // replace key for this entry with JOSM URL
589 eliUrls.remove(urle);
590 eliUrls.put(urlj, e);
591 break;
592 }
593 Collection<String> old = j.getOldIds();
594 if (old != null) {
595 for (String oidj : old) {
596 if (ide.equals(oidj) && Objects.equals(getType(j), getType(e))) {
597 if (getType(j).equals("wms") && unifyWMS(urle).equals(unifyWMS(urlj))) {
598 myprintln("# WMS-URL for oldid "+idj+" modified: "+getDescription(j));
599 } else {
600 myprintln("* URL for oldid "+idj+" differs ("+urle+"): "+getDescription(j));
601 }
602 le.remove(urle);
603 lj.remove(urlj);
604 // replace key for this entry with JOSM URL
605 eliUrls.remove(urle);
606 eliUrls.put(urlj, e);
607 break;
608 }
609 }
610 }
611 }
612 }
613 }
614 }
615
616 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
617 Collections.sort(le);
618 if (!le.isEmpty()) {
619 for (String l : le) {
620 myprintln("- " + getDescription(eliUrls.get(l)));
621 }
622 }
623 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
624 Collections.sort(lj);
625 if (!lj.isEmpty()) {
626 for (String l : lj) {
627 myprintln("+ " + getDescription(josmUrls.get(l)));
628 }
629 }
630 }
631
632 void checkCommonEntries() {
633 doSameUrlButDifferentName();
634 doSameUrlButDifferentId();
635 doSameUrlButDifferentType();
636 doSameUrlButDifferentZoomBounds();
637 doSameUrlButDifferentCountryCode();
638 doSameUrlButDifferentQuality();
639 doSameUrlButDifferentDates();
640 doSameUrlButDifferentInformation();
641 doMismatchingShapes();
642 doMismatchingIcons();
643 doMismatchingCategories();
644 doMiscellaneousChecks();
645 }
646
647 void doSameUrlButDifferentName() {
648 myprintln("*** Same URL, but different name: ***");
649 for (String url : eliUrls.keySet()) {
650 JsonObject e = eliUrls.get(url);
651 if (!josmUrls.containsKey(url)) continue;
652 ImageryInfo j = josmUrls.get(url);
653 String ename = getName(e).replace("'", "\u2019");
654 String jname = getName(j).replace("'", "\u2019");
655 if (!ename.equals(jname)) {
656 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
657 }
658 }
659 }
660
661 void doSameUrlButDifferentId() {
662 myprintln("*** Same URL, but different Id: ***");
663 for (String url : eliUrls.keySet()) {
664 JsonObject e = eliUrls.get(url);
665 if (!josmUrls.containsKey(url)) continue;
666 ImageryInfo j = josmUrls.get(url);
667 String ename = getId(e);
668 String jname = getId(j);
669 if (!Objects.equals(ename, jname)) {
670 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
671 }
672 }
673 }
674
675 void doSameUrlButDifferentType() {
676 myprintln("*** Same URL, but different type: ***");
677 for (String url : eliUrls.keySet()) {
678 JsonObject e = eliUrls.get(url);
679 if (!josmUrls.containsKey(url)) continue;
680 ImageryInfo j = josmUrls.get(url);
681 if (!Objects.equals(getType(e), getType(j))) {
682 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
683 }
684 }
685 }
686
687 void doSameUrlButDifferentZoomBounds() {
688 myprintln("*** Same URL, but different zoom bounds: ***");
689 for (String url : eliUrls.keySet()) {
690 JsonObject e = eliUrls.get(url);
691 if (!josmUrls.containsKey(url)) continue;
692 ImageryInfo j = josmUrls.get(url);
693
694 Integer eMinZoom = getMinZoom(e);
695 Integer jMinZoom = getMinZoom(j);
696 /* dont warn for entries copied from the base of the mirror */
697 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
698 jMinZoom = null;
699 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
700 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
701 }
702 Integer eMaxZoom = getMaxZoom(e);
703 Integer jMaxZoom = getMaxZoom(j);
704 /* dont warn for entries copied from the base of the mirror */
705 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
706 jMaxZoom = null;
707 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
708 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
709 }
710 }
711 }
712
713 void doSameUrlButDifferentCountryCode() {
714 myprintln("*** Same URL, but different country code: ***");
715 for (String url : eliUrls.keySet()) {
716 JsonObject e = eliUrls.get(url);
717 if (!josmUrls.containsKey(url)) continue;
718 ImageryInfo j = josmUrls.get(url);
719 String cce = getCountryCode(e);
720 if ("ZZ".equals(cce)) { /* special ELI country code */
721 cce = null;
722 }
723 if (cce != null && !cce.equals(getCountryCode(j))) {
724 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
725 }
726 }
727 }
728
729 void doSameUrlButDifferentQuality() {
730 myprintln("*** Same URL, but different quality: ***");
731 for (String url : eliUrls.keySet()) {
732 JsonObject e = eliUrls.get(url);
733 if (!josmUrls.containsKey(url)) {
734 String q = getQuality(e);
735 if ("eli-best".equals(q)) {
736 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
737 }
738 continue;
739 }
740 ImageryInfo j = josmUrls.get(url);
741 if (!Objects.equals(getQuality(e), getQuality(j))) {
742 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
743 }
744 }
745 }
746
747 void doSameUrlButDifferentDates() {
748 myprintln("*** Same URL, but different dates: ***");
749 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
750 for (String url : eliUrls.keySet()) {
751 String ed = getDate(eliUrls.get(url));
752 if (!josmUrls.containsKey(url)) continue;
753 ImageryInfo j = josmUrls.get(url);
754 String jd = getDate(j);
755 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
756 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
757 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
758 String ed2 = ed;
759 Matcher m = pattern.matcher(ed);
760 if (m.matches()) {
761 Calendar cal = Calendar.getInstance();
762 cal.set(Integer.valueOf(m.group(2)),
763 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
764 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
765 cal.add(Calendar.DAY_OF_MONTH, -1);
766 ed2 = m.group(1) + cal.get(Calendar.YEAR);
767 if (m.group(4) != null)
768 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
769 if (m.group(6) != null)
770 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
771 }
772 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
773 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
774 String t = "'"+ed+"'";
775 if (!ed.equals(ef)) {
776 t += " or '"+ef+"'";
777 }
778 if (jd.isEmpty()) {
779 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
780 } else if (!ed.isEmpty()) {
781 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
782 } else if (!optionNoEli) {
783 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
784 }
785 }
786 }
787 }
788
789 void doSameUrlButDifferentInformation() {
790 myprintln("*** Same URL, but different information: ***");
791 for (String url : eliUrls.keySet()) {
792 if (!josmUrls.containsKey(url)) continue;
793 JsonObject e = eliUrls.get(url);
794 ImageryInfo j = josmUrls.get(url);
795
796 compareDescriptions(e, j);
797 comparePrivacyPolicyUrls(e, j);
798 comparePermissionReferenceUrls(e, j);
799 compareAttributionUrls(e, j);
800 compareAttributionTexts(e, j);
801 compareProjections(e, j);
802 compareDefaults(e, j);
803 compareOverlays(e, j);
804 compareNoTileHeaders(e, j);
805 }
806 }
807
808 void compareDescriptions(JsonObject e, ImageryInfo j) {
809 String et = getDescriptions(e).getOrDefault("en", "");
810 String jt = getDescriptions(j).getOrDefault("en", "");
811 if (!et.equals(jt)) {
812 if (jt.isEmpty()) {
813 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
814 } else if (!et.isEmpty()) {
815 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
816 } else if (!optionNoEli) {
817 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
818 }
819 }
820 }
821
822 void comparePrivacyPolicyUrls(JsonObject e, ImageryInfo j) {
823 String et = getPrivacyPolicyUrl(e);
824 String jt = getPrivacyPolicyUrl(j);
825 if (!Objects.equals(et, jt)) {
826 if (isBlank(jt)) {
827 myprintln("- Missing JOSM privacy policy URL ("+et+"): "+getDescription(j));
828 } else if (isNotBlank(et)) {
829 myprintln("* Privacy policy URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
830 } else if (!optionNoEli) {
831 myprintln("+ Missing ELI privacy policy URL ('"+jt+"'): "+getDescription(j));
832 }
833 }
834 }
835
836 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
837 String et = getPermissionReferenceUrl(e);
838 String jt = getPermissionReferenceUrl(j);
839 String jt2 = getTermsOfUseUrl(j);
840 if (isBlank(jt)) jt = jt2;
841 if (!Objects.equals(et, jt)) {
842 if (isBlank(jt)) {
843 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
844 } else if (isNotBlank(et)) {
845 String ethttps = et.replace("http:", "https:");
846 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
847 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
848 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
849 } else {
850 String ja = getAttributionUrl(j);
851 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
852 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
853 } else {
854 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
855 }
856 }
857 }
858 } else if (!optionNoEli) {
859 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
860 }
861 }
862 }
863
864 void compareAttributionUrls(JsonObject e, ImageryInfo j) {
865 String et = getAttributionUrl(e);
866 String jt = getAttributionUrl(j);
867 if (!Objects.equals(et, jt)) {
868 if (isBlank(jt)) {
869 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
870 } else if (isNotBlank(et)) {
871 String ethttps = et.replace("http:", "https:");
872 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
873 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
874 } else {
875 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
876 }
877 } else if (!optionNoEli) {
878 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
879 }
880 }
881 }
882
883 void compareAttributionTexts(JsonObject e, ImageryInfo j) {
884 String et = getAttributionText(e);
885 String jt = getAttributionText(j);
886 if (!Objects.equals(et, jt)) {
887 if (isBlank(jt)) {
888 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
889 } else if (isNotBlank(et)) {
890 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
891 } else if (!optionNoEli) {
892 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
893 }
894 }
895 }
896
897 void compareProjections(JsonObject e, ImageryInfo j) {
898 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
899 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
900 if (!Objects.equals(et, jt)) {
901 if (isBlank(jt)) {
902 String t = getType(e);
903 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
904 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
905 } else {
906 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
907 }
908 } else if (isNotBlank(et)) {
909 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
910 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
911 } else {
912 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
913 }
914 } else if (!optionNoEli && !"tms".equals(getType(e))) {
915 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
916 }
917 }
918 }
919
920 void compareDefaults(JsonObject e, ImageryInfo j) {
921 boolean ed = getDefault(e);
922 boolean jd = getDefault(j);
923 if (ed != jd) {
924 if (!jd) {
925 myprintln("- Missing JOSM default: "+getDescription(j));
926 } else if (!optionNoEli) {
927 myprintln("+ Missing ELI default: "+getDescription(j));
928 }
929 }
930 }
931
932 void compareOverlays(JsonObject e, ImageryInfo j) {
933 boolean eo = getOverlay(e);
934 boolean jo = getOverlay(j);
935 if (eo != jo) {
936 if (!jo) {
937 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
938 } else if (!optionNoEli) {
939 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
940 }
941 }
942 }
943
944 void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
945 Map<String, Set<String>> eh = getNoTileHeader(e);
946 Map<String, Set<String>> jh = getNoTileHeader(j);
947 if (!Objects.equals(eh, jh)) {
948 if (jh == null || jh.isEmpty()) {
949 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
950 } else if (eh != null && !eh.isEmpty()) {
951 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
952 } else if (!optionNoEli) {
953 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
954 }
955 }
956 }
957
958 void doMismatchingShapes() {
959 myprintln("*** Mismatching shapes: ***");
960 for (String url : josmUrls.keySet()) {
961 ImageryInfo j = josmUrls.get(url);
962 int num = 1;
963 for (Shape shape : getShapes(j)) {
964 List<Coordinate> p = shape.getPoints();
965 if (!p.get(0).equals(p.get(p.size()-1))) {
966 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
967 }
968 for (int nump = 1; nump < p.size(); ++nump) {
969 if (Objects.equals(p.get(nump-1), p.get(nump))) {
970 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
971 }
972 }
973 ++num;
974 }
975 }
976 for (String url : eliUrls.keySet()) {
977 JsonObject e = eliUrls.get(url);
978 int num = 1;
979 List<Shape> s = null;
980 try {
981 s = getShapes(e);
982 for (Shape shape : s) {
983 List<Coordinate> p = shape.getPoints();
984 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
985 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
986 }
987 for (int nump = 1; nump < p.size(); ++nump) {
988 if (Objects.equals(p.get(nump-1), p.get(nump))) {
989 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
990 }
991 }
992 ++num;
993 }
994 } catch (IllegalArgumentException err) {
995 String desc = getDescription(e);
996 myprintln("+++ ELI shape contains invalid data for "+desc+": "+err.getMessage());
997 }
998 if (s == null || !josmUrls.containsKey(url)) {
999 continue;
1000 }
1001 ImageryInfo j = josmUrls.get(url);
1002 List<Shape> js = getShapes(j);
1003 if (s.isEmpty() && !js.isEmpty()) {
1004 if (!optionNoEli) {
1005 myprintln("+ No ELI shape: "+getDescription(j));
1006 }
1007 } else if (js.isEmpty() && !s.isEmpty()) {
1008 // don't report boundary like 5 point shapes as difference
1009 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
1010 myprintln("- No JOSM shape: "+getDescription(j));
1011 }
1012 } else if (s.size() != js.size()) {
1013 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
1014 } else {
1015 boolean[] edone = new boolean[s.size()];
1016 boolean[] jdone = new boolean[js.size()];
1017 for (int enums = 0; enums < s.size(); ++enums) {
1018 List<Coordinate> ep = s.get(enums).getPoints();
1019 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1020 List<Coordinate> jp = js.get(jnums).getPoints();
1021 if (ep.size() == jp.size() && !jdone[jnums]) {
1022 boolean err = false;
1023 for (int nump = 0; nump < ep.size() && !err; ++nump) {
1024 Coordinate ept = ep.get(nump);
1025 Coordinate jpt = jp.get(nump);
1026 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
1027 err = true;
1028 }
1029 if (!err) {
1030 edone[enums] = true;
1031 jdone[jnums] = true;
1032 break;
1033 }
1034 }
1035 }
1036 }
1037 for (int enums = 0; enums < s.size(); ++enums) {
1038 List<Coordinate> ep = s.get(enums).getPoints();
1039 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1040 List<Coordinate> jp = js.get(jnums).getPoints();
1041 if (ep.size() == jp.size() && !jdone[jnums]) {
1042 boolean err = false;
1043 for (int nump = 0; nump < ep.size() && !err; ++nump) {
1044 Coordinate ept = ep.get(nump);
1045 Coordinate jpt = jp.get(nump);
1046 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
1047 String numtxt = Integer.toString(enums+1);
1048 if (enums != jnums) {
1049 numtxt += '/' + Integer.toString(jnums+1);
1050 }
1051 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
1052 break;
1053 }
1054 }
1055 edone[enums] = true;
1056 jdone[jnums] = true;
1057 break;
1058 }
1059 }
1060 }
1061 for (int enums = 0; enums < s.size(); ++enums) {
1062 List<Coordinate> ep = s.get(enums).getPoints();
1063 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1064 List<Coordinate> jp = js.get(jnums).getPoints();
1065 if (!jdone[jnums]) {
1066 String numtxt = Integer.toString(enums+1);
1067 if (enums != jnums) {
1068 numtxt += '/' + Integer.toString(jnums+1);
1069 }
1070 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
1071 + getDescription(j));
1072 edone[enums] = true;
1073 jdone[jnums] = true;
1074 break;
1075 }
1076 }
1077 }
1078 }
1079 }
1080 }
1081
1082 void doMismatchingIcons() {
1083 myprintln("*** Mismatching icons: ***");
1084 doMismatching(this::compareIcons);
1085 }
1086
1087 void doMismatchingCategories() {
1088 myprintln("*** Mismatching categories: ***");
1089 doMismatching(this::compareCategories);
1090 }
1091
1092 void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
1093 for (String url : eliUrls.keySet()) {
1094 if (josmUrls.containsKey(url)) {
1095 comparator.accept(josmUrls.get(url), eliUrls.get(url));
1096 }
1097 }
1098 }
1099
1100 void compareIcons(ImageryInfo j, JsonObject e) {
1101 String ij = getIcon(j);
1102 String ie = getIcon(e);
1103 boolean ijok = isNotBlank(ij);
1104 boolean ieok = isNotBlank(ie);
1105 if (ijok && !ieok) {
1106 if (!optionNoEli) {
1107 myprintln("+ No ELI icon: "+getDescription(j));
1108 }
1109 } else if (!ijok && ieok) {
1110 myprintln("- No JOSM icon: "+getDescription(j));
1111 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1112 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1113 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1114 ij.startsWith("data:"))) {
1115 String iehttps = ie.replace("http:", "https:");
1116 if (ij.equals(iehttps)) {
1117 myprintln("+ Different icons: "+getDescription(j));
1118 } else {
1119 myprintln("* Different icons: "+getDescription(j));
1120 }
1121 }
1122 }
1123
1124 void compareCategories(ImageryInfo j, JsonObject e) {
1125 String cj = getCategory(j);
1126 String ce = getCategory(e);
1127 boolean cjok = isNotBlank(cj);
1128 boolean ceok = isNotBlank(ce);
1129 if (cjok && !ceok) {
1130 if (!optionNoEli) {
1131 myprintln("+ No ELI category: "+getDescription(j));
1132 }
1133 } else if (!cjok && ceok) {
1134 myprintln("- No JOSM category: "+getDescription(j));
1135 } else if (cjok && ceok && !Objects.equals(cj, ce)) {
1136 myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
1137 }
1138 }
1139
1140 void doMiscellaneousChecks() {
1141 myprintln("*** Miscellaneous checks: ***");
1142 Map<String, ImageryInfo> josmIds = new HashMap<>();
1143 Collection<String> all = Projections.getAllProjectionCodes();
1144 DomainValidator dv = DomainValidator.getInstance();
1145 for (String url : josmUrls.keySet()) {
1146 ImageryInfo j = josmUrls.get(url);
1147 String id = getId(j);
1148 if ("wms".equals(getType(j))) {
1149 String urlLc = url.toLowerCase(Locale.ENGLISH);
1150 if (getProjections(j).isEmpty()) {
1151 myprintln("* WMS without projections: "+getDescription(j));
1152 } else {
1153 List<String> unsupported = new LinkedList<>();
1154 List<String> old = new LinkedList<>();
1155 for (String p : getProjectionsUnstripped(j)) {
1156 if ("CRS:84".equals(p)) {
1157 if (!urlLc.contains("version=1.3")) {
1158 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
1159 }
1160 } else if (oldproj.containsKey(p)) {
1161 old.add(p);
1162 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1163 unsupported.add(p);
1164 }
1165 }
1166 if (!unsupported.isEmpty()) {
1167 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1168 }
1169 for (String o : old) {
1170 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1171 + getDescription(j));
1172 }
1173 }
1174 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1175 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1176 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1177 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
1178 }
1179 }
1180 List<String> urls = new LinkedList<>();
1181 if (!"scanex".equals(getType(j))) {
1182 urls.add(url);
1183 }
1184 String jt = getPermissionReferenceUrl(j);
1185 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1186 urls.add(jt);
1187 jt = getTermsOfUseUrl(j);
1188 if (isNotBlank(jt))
1189 urls.add(jt);
1190 jt = getAttributionUrl(j);
1191 if (isNotBlank(jt))
1192 urls.add(jt);
1193 jt = getIcon(j);
1194 if (isNotBlank(jt)) {
1195 if (!jt.startsWith("data:image/"))
1196 urls.add(jt);
1197 else {
1198 try {
1199 new ImageProvider(jt).get();
1200 } catch (RuntimeException e) {
1201 myprintln("* Strange Icon: "+getDescription(j));
1202 }
1203 }
1204 }
1205 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1206 for (String u : urls) {
1207 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1208 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1209 } else {
1210 try {
1211 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1212 String domain = jurl.getHost();
1213 int port = jurl.getPort();
1214 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1215 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1216 else if (80 == port || 443 == port) {
1217 myprintln("* Useless port '"+port+"': "+getDescription(j));
1218 }
1219 } catch (MalformedURLException e) {
1220 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1221 }
1222 }
1223 }
1224
1225 if (josmMirrors.containsKey(url)) {
1226 continue;
1227 }
1228 if (isBlank(id)) {
1229 myprintln("* No JOSM-ID: "+getDescription(j));
1230 } else if (josmIds.containsKey(id)) {
1231 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
1232 } else {
1233 josmIds.put(id, j);
1234 }
1235 String d = getDate(j);
1236 if (isNotBlank(d)) {
1237 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1238 Matcher m = patternD.matcher(d);
1239 if (!m.matches()) {
1240 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
1241 } else {
1242 try {
1243 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1244 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1245 if (second.compareTo(first) < 0) {
1246 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1247 }
1248 } catch (Exception e) {
1249 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1250 }
1251 }
1252 }
1253 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1254 myprintln("* Attribution link without text: "+getDescription(j));
1255 }
1256 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1257 myprintln("* Logo link without image: "+getDescription(j));
1258 }
1259 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1260 myprintln("* Terms of Use text without link: "+getDescription(j));
1261 }
1262 List<Shape> js = getShapes(j);
1263 if (!js.isEmpty()) {
1264 double minlat = 1000;
1265 double minlon = 1000;
1266 double maxlat = -1000;
1267 double maxlon = -1000;
1268 for (Shape s: js) {
1269 for (Coordinate p: s.getPoints()) {
1270 double lat = p.getLat();
1271 double lon = p.getLon();
1272 if (lat > maxlat) maxlat = lat;
1273 if (lon > maxlon) maxlon = lon;
1274 if (lat < minlat) minlat = lat;
1275 if (lon < minlon) minlon = lon;
1276 }
1277 }
1278 ImageryBounds b = j.getBounds();
1279 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1280 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1281 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1282 + getDescription(j));
1283 }
1284 }
1285 List<String> knownCategories = Arrays.asList(
1286 "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
1287 String cat = getCategory(j);
1288 if (isBlank(cat)) {
1289 myprintln("* No category: "+getDescription(j));
1290 } else if (!knownCategories.contains(cat)) {
1291 myprintln("* Strange category "+cat+": "+getDescription(j));
1292 }
1293 }
1294 }
1295
1296 /*
1297 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1298 */
1299
1300 static String getUrl(Object e) {
1301 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1302 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1303 }
1304
1305 static String getUrlStripped(Object e) {
1306 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1307 }
1308
1309 static String getDate(Object e) {
1310 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1311 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1312 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1313 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1314 if (!start.isEmpty() && !end.isEmpty())
1315 return start+";"+end;
1316 else if (!start.isEmpty())
1317 return start+";-";
1318 else if (!end.isEmpty())
1319 return "-;"+end;
1320 return "";
1321 }
1322
1323 static Date verifyDate(String year, String month, String day) throws ParseException {
1324 String date;
1325 if (year == null) {
1326 date = "3000-01-01";
1327 } else {
1328 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1329 }
1330 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1331 df.setLenient(false);
1332 return df.parse(date);
1333 }
1334
1335 static String getId(Object e) {
1336 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1337 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1338 }
1339
1340 static String getName(Object e) {
1341 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1342 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1343 }
1344
1345 static List<ImageryInfo> getMirrors(Object e) {
1346 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1347 return Collections.emptyList();
1348 }
1349
1350 static List<String> getProjections(Object e) {
1351 List<String> r = new ArrayList<>();
1352 List<String> u = getProjectionsUnstripped(e);
1353 if (u != null) {
1354 for (String p : u) {
1355 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1356 r.add(p);
1357 }
1358 }
1359 }
1360 return r;
1361 }
1362
1363 static List<String> getProjectionsUnstripped(Object e) {
1364 List<String> r = null;
1365 if (e instanceof ImageryInfo) {
1366 r = ((ImageryInfo) e).getServerProjections();
1367 } else {
1368 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1369 if (s != null) {
1370 r = new ArrayList<>();
1371 for (JsonValue p : s.asJsonArray()) {
1372 r.add(((JsonString) p).getString());
1373 }
1374 }
1375 }
1376 return r != null ? r : Collections.emptyList();
1377 }
1378
1379 static void addJsonShapes(List<Shape> l, JsonArray a) {
1380 if (a.get(0).asJsonArray().get(0) instanceof JsonArray) {
1381 for (JsonValue sub: a.asJsonArray()) {
1382 addJsonShapes(l, sub.asJsonArray());
1383 }
1384 } else {
1385 Shape s = new Shape();
1386 for (JsonValue point: a.asJsonArray()) {
1387 JsonArray ar = point.asJsonArray();
1388 String lon = ar.getJsonNumber(0).toString();
1389 String lat = ar.getJsonNumber(1).toString();
1390 s.addPoint(lat, lon);
1391 }
1392 l.add(s);
1393 }
1394 }
1395
1396 static List<Shape> getShapes(Object e) {
1397 if (e instanceof ImageryInfo) {
1398 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1399 if (bounds != null) {
1400 return bounds.getShapes();
1401 }
1402 return Collections.emptyList();
1403 }
1404 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1405 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1406 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1407 List<Shape> l = new ArrayList<>();
1408 for (JsonValue shapes: poly) {
1409 addJsonShapes(l, shapes.asJsonArray());
1410 }
1411 return l;
1412 }
1413 return Collections.emptyList();
1414 }
1415
1416 static String getType(Object e) {
1417 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1418 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1419 }
1420
1421 static Integer getMinZoom(Object e) {
1422 if (e instanceof ImageryInfo) {
1423 int mz = ((ImageryInfo) e).getMinZoom();
1424 return mz == 0 ? null : mz;
1425 } else {
1426 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1427 if (num == null) return null;
1428 return num.intValue();
1429 }
1430 }
1431
1432 static Integer getMaxZoom(Object e) {
1433 if (e instanceof ImageryInfo) {
1434 int mz = ((ImageryInfo) e).getMaxZoom();
1435 return mz == 0 ? null : mz;
1436 } else {
1437 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1438 if (num == null) return null;
1439 return num.intValue();
1440 }
1441 }
1442
1443 static String getCountryCode(Object e) {
1444 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1445 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1446 }
1447
1448 static String getQuality(Object e) {
1449 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1450 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1451 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1452 }
1453
1454 static boolean getOverlay(Object e) {
1455 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1456 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1457 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1458 }
1459
1460 static String getIcon(Object e) {
1461 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1462 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1463 }
1464
1465 static String getAttributionText(Object e) {
1466 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1467 try {
1468 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1469 } catch (NullPointerException ex) {
1470 return null;
1471 }
1472 }
1473
1474 static String getAttributionUrl(Object e) {
1475 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1476 try {
1477 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1478 } catch (NullPointerException ex) {
1479 return null;
1480 }
1481 }
1482
1483 static String getTermsOfUseText(Object e) {
1484 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1485 return null;
1486 }
1487
1488 static String getTermsOfUseUrl(Object e) {
1489 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1490 return null;
1491 }
1492
1493 static String getCategory(Object e) {
1494 if (e instanceof ImageryInfo) {
1495 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1496 }
1497 return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
1498 }
1499
1500 static String getLogoImage(Object e) {
1501 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1502 return null;
1503 }
1504
1505 static String getLogoUrl(Object e) {
1506 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1507 return null;
1508 }
1509
1510 static String getPermissionReferenceUrl(Object e) {
1511 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1512 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1513 }
1514
1515 static String getPrivacyPolicyUrl(Object e) {
1516 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPrivacyPolicyURL();
1517 return ((Map<String, JsonObject>) e).get("properties").getString("privacy_policy_url", null);
1518 }
1519
1520 static Map<String, Set<String>> getNoTileHeader(Object e) {
1521 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1522 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1523 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1524 Function.identity(),
1525 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1526 }
1527
1528 static Map<String, String> getDescriptions(Object e) {
1529 Map<String, String> res = new HashMap<>();
1530 if (e instanceof ImageryInfo) {
1531 String a = ((ImageryInfo) e).getDescription();
1532 if (a != null) res.put("en", a);
1533 } else {
1534 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1535 if (a != null) res.put("en", a.replaceAll("''", "'"));
1536 }
1537 return res;
1538 }
1539
1540 static boolean getValidGeoreference(Object e) {
1541 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1542 return false;
1543 }
1544
1545 static boolean getDefault(Object e) {
1546 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1547 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1548 }
1549
1550 String getDescription(Object o) {
1551 String url = getUrl(o);
1552 String cc = getCountryCode(o);
1553 if (cc == null) {
1554 ImageryInfo j = josmUrls.get(url);
1555 if (j != null) cc = getCountryCode(j);
1556 if (cc == null) {
1557 JsonObject e = eliUrls.get(url);
1558 if (e != null) cc = getCountryCode(e);
1559 }
1560 }
1561 if (cc == null) {
1562 cc = "";
1563 } else {
1564 cc = "["+cc+"] ";
1565 }
1566 String name = getName(o);
1567 String id = getId(o);
1568 String d = cc;
1569 if (name != null && !name.isEmpty()) {
1570 d += name;
1571 if (id != null && !id.isEmpty())
1572 d += " ["+id+"]";
1573 } else if (url != null && !url.isEmpty())
1574 d += url;
1575 if (optionShorten) {
1576 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1577 }
1578 return d;
1579 }
1580}
Note: See TracBrowser for help on using the repository browser.