source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 16112

Last change on this file since 16112 was 16098, checked in by simon04, 6 years ago

SyncEditorLayerIndex: use Files.newBufferedReader and Files.newBufferedWriter
  • Property svn:eol-style set to native
File size: 65.0 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.BufferedWriter;
8import java.io.IOException;
9import java.io.OutputStreamWriter;
10import java.io.Writer;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.charset.Charset;
15import java.nio.file.Files;
16import java.nio.file.Paths;
17import java.text.DecimalFormat;
18import java.text.ParseException;
19import java.text.SimpleDateFormat;
20import java.util.ArrayList;
21import java.util.Arrays;
22import java.util.Calendar;
23import java.util.Collection;
24import java.util.Collections;
25import java.util.Date;
26import java.util.HashMap;
27import java.util.LinkedList;
28import java.util.List;
29import java.util.Locale;
30import java.util.Map;
31import java.util.Map.Entry;
32import java.util.Objects;
33import java.util.Set;
34import java.util.function.BiConsumer;
35import java.util.function.Function;
36import java.util.regex.Matcher;
37import java.util.regex.Pattern;
38import java.util.stream.Collectors;
39
40import javax.json.Json;
41import javax.json.JsonArray;
42import javax.json.JsonNumber;
43import javax.json.JsonObject;
44import javax.json.JsonReader;
45import javax.json.JsonString;
46import javax.json.JsonValue;
47
48import org.openstreetmap.gui.jmapviewer.Coordinate;
49import org.openstreetmap.josm.data.Preferences;
50import org.openstreetmap.josm.data.imagery.ImageryInfo;
51import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
52import org.openstreetmap.josm.data.imagery.Shape;
53import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
54import org.openstreetmap.josm.data.preferences.JosmUrls;
55import org.openstreetmap.josm.data.projection.Projections;
56import org.openstreetmap.josm.data.validation.routines.DomainValidator;
57import org.openstreetmap.josm.io.imagery.ImageryReader;
58import org.openstreetmap.josm.spi.preferences.Config;
59import org.openstreetmap.josm.tools.ImageProvider;
60import org.openstreetmap.josm.tools.JosmRuntimeException;
61import org.openstreetmap.josm.tools.Logging;
62import org.openstreetmap.josm.tools.OptionParser;
63import org.openstreetmap.josm.tools.OptionParser.OptionCount;
64import org.openstreetmap.josm.tools.ReflectionUtils;
65import org.xml.sax.SAXException;
66
67/**
68 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
69 * The goal is to keep both lists in sync.
70 *
71 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
72 * provides also a version in the JOSM format, but the GEOJSON is the original source
73 * format, so we read that.
74 *
75 * How to run:
76 * -----------
77 *
78 * Main JOSM binary needs to be in classpath, e.g.
79 *
80 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
81 *
82 * Add option "-h" to show the available command line flags.
83 */
84@SuppressWarnings("unchecked")
85public class SyncEditorLayerIndex {
86
87 private static final int MAXLEN = 140;
88
89 private List<ImageryInfo> josmEntries;
90 private JsonArray eliEntries;
91
92 private final Map<String, JsonObject> eliUrls = new HashMap<>();
93 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
94 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
95 private static final Map<String, String> oldproj = new HashMap<>();
96 private static final List<String> ignoreproj = new LinkedList<>();
97
98 private static String eliInputFile = "imagery_eli.geojson";
99 private static String josmInputFile = "imagery_josm.imagery.xml";
100 private static String ignoreInputFile = "imagery_josm.ignores.txt";
101 private static Writer outputStream;
102 private static String optionOutput;
103 private static boolean optionShorten;
104 private static boolean optionNoSkip;
105 private static boolean optionXhtmlBody;
106 private static boolean optionXhtml;
107 private static String optionEliXml;
108 private static String optionJosmXml;
109 private static String optionEncoding;
110 private static boolean optionNoEli;
111 private Map<String, String> skip = new HashMap<>();
112 private Map<String, String> skipStart = new HashMap<>();
113
114 /**
115 * Main method.
116 * @param args program arguments
117 * @throws IOException if any I/O error occurs
118 * @throws ReflectiveOperationException if any reflective operation error occurs
119 * @throws SAXException if any SAX error occurs
120 */
121 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
122 Locale.setDefault(Locale.ROOT);
123 parseCommandLineArguments(args);
124 Config.setUrlsProvider(JosmUrls.getInstance());
125 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
126 Config.setPreferencesInstance(pref);
127 pref.init(false);
128 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
129 script.setupProj();
130 script.loadSkip();
131 script.start();
132 script.loadJosmEntries();
133 if (optionJosmXml != null) {
134 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionJosmXml), UTF_8)) {
135 script.printentries(script.josmEntries, writer);
136 }
137 }
138 script.loadELIEntries();
139 if (optionEliXml != null) {
140 try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(optionEliXml), UTF_8)) {
141 script.printentries(script.eliEntries, writer);
142 }
143 }
144 script.checkInOneButNotTheOther();
145 script.checkCommonEntries();
146 script.end();
147 if (outputStream != null) {
148 outputStream.close();
149 }
150 }
151
152 /**
153 * Displays help on the console
154 */
155 private static void showHelp() {
156 System.out.println(getHelp());
157 System.exit(0);
158 }
159
160 static String getHelp() {
161 return "usage: java -cp build SyncEditorLayerIndex\n" +
162 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
163 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
164 "Default is imagery_eli.geojson (current directory).\n" +
165 "-h,--help show this help\n" +
166 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
167 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
168 "Default is imagery_josm.imagery.xml (current directory).\n" +
169 "-m,--noeli don't show output for ELI problems\n" +
170 "-n,--noskip don't skip known entries\n" +
171 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
172 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
173 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
174 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
175 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
176 "-X,--xhtml create XHTML for display in a web page\n";
177 }
178
179 /**
180 * Parse command line arguments.
181 * @param args program arguments
182 * @throws IOException in case of I/O error
183 */
184 static void parseCommandLineArguments(String[] args) throws IOException {
185 new OptionParser("JOSM/ELI synchronization script")
186 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
187 .addShortAlias("help", "h")
188 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
189 .addShortAlias("output", "o")
190 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
191 .addShortAlias("eli_input", "e")
192 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
193 .addShortAlias("josm_input", "j")
194 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
195 .addShortAlias("ignore_input", "i")
196 .addFlagParameter("shorten", () -> optionShorten = true)
197 .addShortAlias("shorten", "s")
198 .addFlagParameter("noskip", () -> optionNoSkip = true)
199 .addShortAlias("noskip", "n")
200 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
201 .addShortAlias("xhtmlbody", "x")
202 .addFlagParameter("xhtml", () -> optionXhtml = true)
203 .addShortAlias("xhtml", "X")
204 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
205 .addShortAlias("elixml", "p")
206 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
207 .addShortAlias("josmxml", "q")
208 .addFlagParameter("noeli", () -> optionNoEli = true)
209 .addShortAlias("noeli", "m")
210 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
211 .addShortAlias("encoding", "c")
212 .parseOptionsOrExit(Arrays.asList(args));
213
214 if (optionOutput != null && !"-".equals(optionOutput)) {
215 outputStream = Files.newBufferedWriter(Paths.get(optionOutput), optionEncoding != null ? Charset.forName(optionEncoding) : UTF_8);
216 } else if (optionEncoding != null) {
217 outputStream = new OutputStreamWriter(System.out, optionEncoding);
218 }
219 }
220
221 void setupProj() {
222 oldproj.put("EPSG:3359", "EPSG:3404");
223 oldproj.put("EPSG:3785", "EPSG:3857");
224 oldproj.put("EPSG:31297", "EPGS:31287");
225 oldproj.put("EPSG:31464", "EPSG:31468");
226 oldproj.put("EPSG:54004", "EPSG:3857");
227 oldproj.put("EPSG:102100", "EPSG:3857");
228 oldproj.put("EPSG:102113", "EPSG:3857");
229 oldproj.put("EPSG:900913", "EPGS:3857");
230 ignoreproj.add("EPSG:4267");
231 ignoreproj.add("EPSG:5221");
232 ignoreproj.add("EPSG:5514");
233 ignoreproj.add("EPSG:32019");
234 ignoreproj.add("EPSG:102066");
235 ignoreproj.add("EPSG:102067");
236 ignoreproj.add("EPSG:102685");
237 ignoreproj.add("EPSG:102711");
238 }
239
240 void loadSkip() throws IOException {
241 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
242 try (BufferedReader fr = Files.newBufferedReader(Paths.get(ignoreInputFile), UTF_8)) {
243 String line;
244
245 while ((line = fr.readLine()) != null) {
246 Matcher res = pattern.matcher(line);
247 if (res.matches()) {
248 String s = res.group(2);
249 if (s.endsWith("...")) {
250 s = s.substring(0, s.length() - 3);
251 if ("Ignore".equals(res.group(1))) {
252 skipStart.put(s, "green");
253 } else {
254 skipStart.put(s, "darkgoldenrod");
255 }
256 } else {
257 if ("Ignore".equals(res.group(1))) {
258 skip.put(s, "green");
259 } else {
260 skip.put(s, "darkgoldenrod");
261 }
262 }
263 }
264 }
265 }
266 }
267
268 void myprintlnfinal(String s) {
269 if (outputStream != null) {
270 try {
271 outputStream.write(s + System.getProperty("line.separator"));
272 } catch (IOException e) {
273 throw new JosmRuntimeException(e);
274 }
275 } else {
276 System.out.println(s);
277 }
278 }
279
280 String isSkipString(String s) {
281 if (skip.containsKey(s))
282 return skip.get(s);
283 for (Entry<String, String> str : skipStart.entrySet()) {
284 if (s.startsWith(str.getKey())) {
285 skipStart.remove(str.getKey());
286 return str.getValue();
287 }
288 }
289 return null;
290 }
291
292 void myprintln(String s) {
293 String color;
294 if ((color = isSkipString(s)) != null) {
295 skip.remove(s);
296 if (optionXhtmlBody || optionXhtml) {
297 s = "<pre style=\"margin:3px;color:"+color+"\">"
298 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
299 }
300 if (!optionNoSkip) {
301 return;
302 }
303 } else if (optionXhtmlBody || optionXhtml) {
304 color =
305 s.startsWith("***") ? "black" :
306 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
307 (s.startsWith("#") ? "indigo" :
308 (s.startsWith("!") ? "orange" : "red")));
309 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
310 }
311 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
312 return;
313 }
314 myprintlnfinal(s);
315 }
316
317 void start() {
318 if (optionXhtml) {
319 myprintlnfinal(
320 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
321 myprintlnfinal(
322 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
323 "<title>JOSM - ELI differences</title></head><body>\n");
324 }
325 }
326
327 void end() {
328 for (String s : skip.keySet()) {
329 myprintln("+++ Obsolete skip entry: " + s);
330 }
331 for (String s : skipStart.keySet()) {
332 myprintln("+++ Obsolete skip entry: " + s + "...");
333 }
334 if (optionXhtml) {
335 myprintlnfinal("</body></html>\n");
336 }
337 }
338
339 void loadELIEntries() throws IOException {
340 try (JsonReader jr = Json.createReader(Files.newBufferedReader(Paths.get(eliInputFile), UTF_8))) {
341 eliEntries = jr.readObject().getJsonArray("features");
342 }
343
344 for (JsonValue e : eliEntries) {
345 String url = getUrlStripped(e);
346 if (url.contains("{z}")) {
347 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+getDescription(e));
348 url = url.replace("{z}", "{zoom}");
349 }
350 if (eliUrls.containsKey(url)) {
351 myprintln("+++ ELI-URL is not unique: "+url);
352 } else {
353 eliUrls.put(url, e.asJsonObject());
354 }
355 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
356 if (s != null) {
357 String urlLc = url.toLowerCase(Locale.ENGLISH);
358 List<String> old = new LinkedList<>();
359 for (JsonValue p : s) {
360 String proj = ((JsonString) p).getString();
361 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
362 old.add(proj);
363 }
364 }
365 if (!old.isEmpty()) {
366 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
367 }
368 }
369 }
370 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
371 }
372
373 String cdata(String s) {
374 return cdata(s, false);
375 }
376
377 String cdata(String s, boolean escape) {
378 if (escape) {
379 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
380 } else if (s.matches(".*[<>&].*"))
381 return "<![CDATA["+s+"]]>";
382 return s;
383 }
384
385 String maininfo(Object entry, String offset) {
386 String t = getType(entry);
387 String res = offset + "<type>"+t+"</type>\n";
388 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
389 if (getMinZoom(entry) != null)
390 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
391 if (getMaxZoom(entry) != null)
392 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
393 if ("wms".equals(t)) {
394 List<String> p = getProjections(entry);
395 if (p != null) {
396 res += offset + "<projections>\n";
397 for (String c : p) {
398 res += offset + " <code>"+c+"</code>\n";
399 }
400 res += offset + "</projections>\n";
401 }
402 }
403 return res;
404 }
405
406 void printentries(List<?> entries, Writer stream) throws IOException {
407 DecimalFormat df = new DecimalFormat("#.#######");
408 df.setRoundingMode(java.math.RoundingMode.CEILING);
409 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
410 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
411 for (Object e : entries) {
412 stream.write(" <entry"
413 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
414 + (getOverlay(e) ? " overlay=\"true\"" : "")
415 + ">\n");
416 String t;
417 if (isNotBlank(t = getName(e)))
418 stream.write(" <name>"+cdata(t, true)+"</name>\n");
419 if (isNotBlank(t = getId(e)))
420 stream.write(" <id>"+t+"</id>\n");
421 if (isNotBlank(t = getCategory(e)))
422 stream.write(" <category>"+t+"</category>\n");
423 if (isNotBlank(t = getDate(e)))
424 stream.write(" <date>"+t+"</date>\n");
425 if (isNotBlank(t = getCountryCode(e)))
426 stream.write(" <country-code>"+t+"</country-code>\n");
427 if ((getDefault(e)))
428 stream.write(" <default>true</default>\n");
429 stream.write(maininfo(e, " "));
430 if (isNotBlank(t = getAttributionText(e)))
431 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
432 if (isNotBlank(t = getAttributionUrl(e)))
433 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
434 if (isNotBlank(t = getLogoImage(e)))
435 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
436 if (isNotBlank(t = getLogoUrl(e)))
437 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
438 if (isNotBlank(t = getTermsOfUseText(e)))
439 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
440 if (isNotBlank(t = getTermsOfUseUrl(e)))
441 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
442 if (isNotBlank(t = getPermissionReferenceUrl(e)))
443 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
444 if ((getValidGeoreference(e)))
445 stream.write(" <valid-georeference>true</valid-georeference>\n");
446 if (isNotBlank(t = getIcon(e)))
447 stream.write(" <icon>"+cdata(t)+"</icon>\n");
448 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
449 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
450 }
451 for (ImageryInfo m : getMirrors(e)) {
452 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
453 }
454 double minlat = 1000;
455 double minlon = 1000;
456 double maxlat = -1000;
457 double maxlon = -1000;
458 String shapes = "";
459 String sep = "\n ";
460 try {
461 for (Shape s: getShapes(e)) {
462 shapes += " <shape>";
463 int i = 0;
464 for (Coordinate p: s.getPoints()) {
465 double lat = p.getLat();
466 double lon = p.getLon();
467 if (lat > maxlat) maxlat = lat;
468 if (lon > maxlon) maxlon = lon;
469 if (lat < minlat) minlat = lat;
470 if (lon < minlon) minlon = lon;
471 if ((i++ % 3) == 0) {
472 shapes += sep + " ";
473 }
474 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
475 }
476 shapes += sep + "</shape>\n";
477 }
478 } catch (IllegalArgumentException ignored) {
479 Logging.trace(ignored);
480 }
481 if (!shapes.isEmpty()) {
482 stream.write(" <bounds min-lat='"+df.format(minlat)
483 +"' min-lon='"+df.format(minlon)
484 +"' max-lat='"+df.format(maxlat)
485 +"' max-lon='"+df.format(maxlon)+"'>\n");
486 stream.write(shapes + " </bounds>\n");
487 }
488 stream.write(" </entry>\n");
489 }
490 stream.write("</imagery>\n");
491 stream.close();
492 }
493
494 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
495 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
496 josmEntries = reader.parse();
497 }
498
499 for (ImageryInfo e : josmEntries) {
500 if (isBlank(getUrl(e))) {
501 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
502 continue;
503 }
504 if (isBlank(getName(e))) {
505 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
506 continue;
507 }
508 String url = getUrlStripped(e);
509 if (url.contains("{z}")) {
510 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+getDescription(e));
511 url = url.replace("{z}", "{zoom}");
512 }
513 if (josmUrls.containsKey(url)) {
514 myprintln("+++ JOSM-URL is not unique: "+url);
515 } else {
516 josmUrls.put(url, e);
517 }
518 for (ImageryInfo m : e.getMirrors()) {
519 url = getUrlStripped(m);
520 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
521 ReflectionUtils.setObjectsAccessible(origNameField);
522 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
523 if (josmUrls.containsKey(url)) {
524 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
525 } else {
526 josmUrls.put(url, m);
527 josmMirrors.put(url, m);
528 }
529 }
530 }
531 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
532 }
533
534 void checkInOneButNotTheOther() {
535 List<String> le = new LinkedList<>(eliUrls.keySet());
536 List<String> lj = new LinkedList<>(josmUrls.keySet());
537
538 List<String> ke = new LinkedList<>(le);
539 for (String url : ke) {
540 if (lj.contains(url)) {
541 le.remove(url);
542 lj.remove(url);
543 }
544 }
545
546 if (!le.isEmpty() && !lj.isEmpty()) {
547 ke = new LinkedList<>(le);
548 for (String urle : ke) {
549 JsonObject e = eliUrls.get(urle);
550 String ide = getId(e);
551 String urlhttps = urle.replace("http:", "https:");
552 if (lj.contains(urlhttps)) {
553 myprintln("+ Missing https: "+getDescription(e));
554 eliUrls.put(urlhttps, eliUrls.get(urle));
555 eliUrls.remove(urle);
556 le.remove(urle);
557 lj.remove(urlhttps);
558 } else if (isNotBlank(ide)) {
559 List<String> kj = new LinkedList<>(lj);
560 for (String urlj : kj) {
561 ImageryInfo j = josmUrls.get(urlj);
562 String idj = getId(j);
563
564 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
565 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
566 le.remove(urle);
567 lj.remove(urlj);
568 // replace key for this entry with JOSM URL
569 eliUrls.remove(urle);
570 eliUrls.put(urlj, e);
571 break;
572 }
573 }
574 }
575 }
576 }
577
578 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
579 Collections.sort(le);
580 if (!le.isEmpty()) {
581 for (String l : le) {
582 myprintln("- " + getDescription(eliUrls.get(l)));
583 }
584 }
585 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
586 Collections.sort(lj);
587 if (!lj.isEmpty()) {
588 for (String l : lj) {
589 myprintln("+ " + getDescription(josmUrls.get(l)));
590 }
591 }
592 }
593
594 void checkCommonEntries() {
595 doSameUrlButDifferentName();
596 doSameUrlButDifferentId();
597 doSameUrlButDifferentType();
598 doSameUrlButDifferentZoomBounds();
599 doSameUrlButDifferentCountryCode();
600 doSameUrlButDifferentQuality();
601 doSameUrlButDifferentDates();
602 doSameUrlButDifferentInformation();
603 doMismatchingShapes();
604 doMismatchingIcons();
605 doMismatchingCategories();
606 doMiscellaneousChecks();
607 }
608
609 void doSameUrlButDifferentName() {
610 myprintln("*** Same URL, but different name: ***");
611 for (String url : eliUrls.keySet()) {
612 JsonObject e = eliUrls.get(url);
613 if (!josmUrls.containsKey(url)) continue;
614 ImageryInfo j = josmUrls.get(url);
615 String ename = getName(e).replace("'", "\u2019");
616 String jname = getName(j).replace("'", "\u2019");
617 if (!ename.equals(jname)) {
618 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
619 }
620 }
621 }
622
623 void doSameUrlButDifferentId() {
624 myprintln("*** Same URL, but different Id: ***");
625 for (String url : eliUrls.keySet()) {
626 JsonObject e = eliUrls.get(url);
627 if (!josmUrls.containsKey(url)) continue;
628 ImageryInfo j = josmUrls.get(url);
629 String ename = getId(e);
630 String jname = getId(j);
631 if (!Objects.equals(ename, jname)) {
632 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
633 }
634 }
635 }
636
637 void doSameUrlButDifferentType() {
638 myprintln("*** Same URL, but different type: ***");
639 for (String url : eliUrls.keySet()) {
640 JsonObject e = eliUrls.get(url);
641 if (!josmUrls.containsKey(url)) continue;
642 ImageryInfo j = josmUrls.get(url);
643 if (!Objects.equals(getType(e), getType(j))) {
644 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
645 }
646 }
647 }
648
649 void doSameUrlButDifferentZoomBounds() {
650 myprintln("*** Same URL, but different zoom bounds: ***");
651 for (String url : eliUrls.keySet()) {
652 JsonObject e = eliUrls.get(url);
653 if (!josmUrls.containsKey(url)) continue;
654 ImageryInfo j = josmUrls.get(url);
655
656 Integer eMinZoom = getMinZoom(e);
657 Integer jMinZoom = getMinZoom(j);
658 /* dont warn for entries copied from the base of the mirror */
659 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
660 jMinZoom = null;
661 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
662 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
663 }
664 Integer eMaxZoom = getMaxZoom(e);
665 Integer jMaxZoom = getMaxZoom(j);
666 /* dont warn for entries copied from the base of the mirror */
667 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
668 jMaxZoom = null;
669 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
670 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
671 }
672 }
673 }
674
675 void doSameUrlButDifferentCountryCode() {
676 myprintln("*** Same URL, but different country code: ***");
677 for (String url : eliUrls.keySet()) {
678 JsonObject e = eliUrls.get(url);
679 if (!josmUrls.containsKey(url)) continue;
680 ImageryInfo j = josmUrls.get(url);
681 String cce = getCountryCode(e);
682 if ("ZZ".equals(cce)) { /* special ELI country code */
683 cce = null;
684 }
685 if (cce != null && !cce.equals(getCountryCode(j))) {
686 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
687 }
688 }
689 }
690
691 void doSameUrlButDifferentQuality() {
692 myprintln("*** Same URL, but different quality: ***");
693 for (String url : eliUrls.keySet()) {
694 JsonObject e = eliUrls.get(url);
695 if (!josmUrls.containsKey(url)) {
696 String q = getQuality(e);
697 if ("eli-best".equals(q)) {
698 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
699 }
700 continue;
701 }
702 ImageryInfo j = josmUrls.get(url);
703 if (!Objects.equals(getQuality(e), getQuality(j))) {
704 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
705 }
706 }
707 }
708
709 void doSameUrlButDifferentDates() {
710 myprintln("*** Same URL, but different dates: ***");
711 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
712 for (String url : eliUrls.keySet()) {
713 String ed = getDate(eliUrls.get(url));
714 if (!josmUrls.containsKey(url)) continue;
715 ImageryInfo j = josmUrls.get(url);
716 String jd = getDate(j);
717 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
718 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
719 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
720 String ed2 = ed;
721 Matcher m = pattern.matcher(ed);
722 if (m.matches()) {
723 Calendar cal = Calendar.getInstance();
724 cal.set(Integer.valueOf(m.group(2)),
725 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
726 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
727 cal.add(Calendar.DAY_OF_MONTH, -1);
728 ed2 = m.group(1) + cal.get(Calendar.YEAR);
729 if (m.group(4) != null)
730 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
731 if (m.group(6) != null)
732 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
733 }
734 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
735 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
736 String t = "'"+ed+"'";
737 if (!ed.equals(ef)) {
738 t += " or '"+ef+"'";
739 }
740 if (jd.isEmpty()) {
741 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
742 } else if (!ed.isEmpty()) {
743 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
744 } else if (!optionNoEli) {
745 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
746 }
747 }
748 }
749 }
750
751 void doSameUrlButDifferentInformation() {
752 myprintln("*** Same URL, but different information: ***");
753 for (String url : eliUrls.keySet()) {
754 if (!josmUrls.containsKey(url)) continue;
755 JsonObject e = eliUrls.get(url);
756 ImageryInfo j = josmUrls.get(url);
757
758 compareDescriptions(e, j);
759 comparePermissionReferenceUrls(e, j);
760 compareAttributionUrls(e, j);
761 compareAttributionTexts(e, j);
762 compareProjections(e, j);
763 compareDefaults(e, j);
764 compareOverlays(e, j);
765 compareNoTileHeaders(e, j);
766 }
767 }
768
769 void compareDescriptions(JsonObject e, ImageryInfo j) {
770 String et = getDescriptions(e).getOrDefault("en", "");
771 String jt = getDescriptions(j).getOrDefault("en", "");
772 if (!et.equals(jt)) {
773 if (jt.isEmpty()) {
774 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
775 } else if (!et.isEmpty()) {
776 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
777 } else if (!optionNoEli) {
778 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
779 }
780 }
781 }
782
783 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) {
784 String et = getPermissionReferenceUrl(e);
785 String jt = getPermissionReferenceUrl(j);
786 String jt2 = getTermsOfUseUrl(j);
787 if (isBlank(jt)) jt = jt2;
788 if (!Objects.equals(et, jt)) {
789 if (isBlank(jt)) {
790 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
791 } else if (isNotBlank(et)) {
792 String ethttps = et.replace("http:", "https:");
793 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
794 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
795 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
796 } else {
797 String ja = getAttributionUrl(j);
798 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
799 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
800 } else {
801 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
802 }
803 }
804 }
805 } else if (!optionNoEli) {
806 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
807 }
808 }
809 }
810
811 void compareAttributionUrls(JsonObject e, ImageryInfo j) {
812 String et = getAttributionUrl(e);
813 String jt = getAttributionUrl(j);
814 if (!Objects.equals(et, jt)) {
815 if (isBlank(jt)) {
816 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
817 } else if (isNotBlank(et)) {
818 String ethttps = et.replace("http:", "https:");
819 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
820 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
821 } else {
822 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
823 }
824 } else if (!optionNoEli) {
825 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
826 }
827 }
828 }
829
830 void compareAttributionTexts(JsonObject e, ImageryInfo j) {
831 String et = getAttributionText(e);
832 String jt = getAttributionText(j);
833 if (!Objects.equals(et, jt)) {
834 if (isBlank(jt)) {
835 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
836 } else if (isNotBlank(et)) {
837 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
838 } else if (!optionNoEli) {
839 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
840 }
841 }
842 }
843
844 void compareProjections(JsonObject e, ImageryInfo j) {
845 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
846 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
847 if (!Objects.equals(et, jt)) {
848 if (isBlank(jt)) {
849 String t = getType(e);
850 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
851 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
852 } else {
853 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
854 }
855 } else if (isNotBlank(et)) {
856 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
857 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
858 } else {
859 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
860 }
861 } else if (!optionNoEli && !"tms".equals(getType(e))) {
862 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
863 }
864 }
865 }
866
867 void compareDefaults(JsonObject e, ImageryInfo j) {
868 boolean ed = getDefault(e);
869 boolean jd = getDefault(j);
870 if (ed != jd) {
871 if (!jd) {
872 myprintln("- Missing JOSM default: "+getDescription(j));
873 } else if (!optionNoEli) {
874 myprintln("+ Missing ELI default: "+getDescription(j));
875 }
876 }
877 }
878
879 void compareOverlays(JsonObject e, ImageryInfo j) {
880 boolean eo = getOverlay(e);
881 boolean jo = getOverlay(j);
882 if (eo != jo) {
883 if (!jo) {
884 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
885 } else if (!optionNoEli) {
886 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
887 }
888 }
889 }
890
891 void compareNoTileHeaders(JsonObject e, ImageryInfo j) {
892 Map<String, Set<String>> eh = getNoTileHeader(e);
893 Map<String, Set<String>> jh = getNoTileHeader(j);
894 if (!Objects.equals(eh, jh)) {
895 if (jh == null || jh.isEmpty()) {
896 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
897 } else if (eh != null && !eh.isEmpty()) {
898 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
899 } else if (!optionNoEli) {
900 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
901 }
902 }
903 }
904
905 void doMismatchingShapes() {
906 myprintln("*** Mismatching shapes: ***");
907 for (String url : josmUrls.keySet()) {
908 ImageryInfo j = josmUrls.get(url);
909 int num = 1;
910 for (Shape shape : getShapes(j)) {
911 List<Coordinate> p = shape.getPoints();
912 if (!p.get(0).equals(p.get(p.size()-1))) {
913 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
914 }
915 for (int nump = 1; nump < p.size(); ++nump) {
916 if (Objects.equals(p.get(nump-1), p.get(nump))) {
917 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
918 }
919 }
920 ++num;
921 }
922 }
923 for (String url : eliUrls.keySet()) {
924 JsonObject e = eliUrls.get(url);
925 int num = 1;
926 List<Shape> s = null;
927 try {
928 s = getShapes(e);
929 for (Shape shape : s) {
930 List<Coordinate> p = shape.getPoints();
931 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
932 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
933 }
934 for (int nump = 1; nump < p.size(); ++nump) {
935 if (Objects.equals(p.get(nump-1), p.get(nump))) {
936 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
937 }
938 }
939 ++num;
940 }
941 } catch (IllegalArgumentException err) {
942 String desc = getDescription(e);
943 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
944 }
945 if (s == null || !josmUrls.containsKey(url)) {
946 continue;
947 }
948 ImageryInfo j = josmUrls.get(url);
949 List<Shape> js = getShapes(j);
950 if (s.isEmpty() && !js.isEmpty()) {
951 if (!optionNoEli) {
952 myprintln("+ No ELI shape: "+getDescription(j));
953 }
954 } else if (js.isEmpty() && !s.isEmpty()) {
955 // don't report boundary like 5 point shapes as difference
956 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
957 myprintln("- No JOSM shape: "+getDescription(j));
958 }
959 } else if (s.size() != js.size()) {
960 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
961 } else {
962 boolean[] edone = new boolean[s.size()];
963 boolean[] jdone = new boolean[js.size()];
964 for (int enums = 0; enums < s.size(); ++enums) {
965 List<Coordinate> ep = s.get(enums).getPoints();
966 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
967 List<Coordinate> jp = js.get(jnums).getPoints();
968 if (ep.size() == jp.size() && !jdone[jnums]) {
969 boolean err = false;
970 for (int nump = 0; nump < ep.size() && !err; ++nump) {
971 Coordinate ept = ep.get(nump);
972 Coordinate jpt = jp.get(nump);
973 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
974 err = true;
975 }
976 if (!err) {
977 edone[enums] = true;
978 jdone[jnums] = true;
979 break;
980 }
981 }
982 }
983 }
984 for (int enums = 0; enums < s.size(); ++enums) {
985 List<Coordinate> ep = s.get(enums).getPoints();
986 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
987 List<Coordinate> jp = js.get(jnums).getPoints();
988 if (ep.size() == jp.size() && !jdone[jnums]) {
989 boolean err = false;
990 for (int nump = 0; nump < ep.size() && !err; ++nump) {
991 Coordinate ept = ep.get(nump);
992 Coordinate jpt = jp.get(nump);
993 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
994 String numtxt = Integer.toString(enums+1);
995 if (enums != jnums) {
996 numtxt += '/' + Integer.toString(jnums+1);
997 }
998 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
999 break;
1000 }
1001 }
1002 edone[enums] = true;
1003 jdone[jnums] = true;
1004 break;
1005 }
1006 }
1007 }
1008 for (int enums = 0; enums < s.size(); ++enums) {
1009 List<Coordinate> ep = s.get(enums).getPoints();
1010 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
1011 List<Coordinate> jp = js.get(jnums).getPoints();
1012 if (!jdone[jnums]) {
1013 String numtxt = Integer.toString(enums+1);
1014 if (enums != jnums) {
1015 numtxt += '/' + Integer.toString(jnums+1);
1016 }
1017 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+"): "
1018 + getDescription(j));
1019 edone[enums] = true;
1020 jdone[jnums] = true;
1021 break;
1022 }
1023 }
1024 }
1025 }
1026 }
1027 }
1028
1029 void doMismatchingIcons() {
1030 myprintln("*** Mismatching icons: ***");
1031 doMismatching(this::compareIcons);
1032 }
1033
1034 void doMismatchingCategories() {
1035 myprintln("*** Mismatching categories: ***");
1036 doMismatching(this::compareCategories);
1037 }
1038
1039 void doMismatching(BiConsumer<ImageryInfo, JsonObject> comparator) {
1040 for (String url : eliUrls.keySet()) {
1041 if (josmUrls.containsKey(url)) {
1042 comparator.accept(josmUrls.get(url), eliUrls.get(url));
1043 }
1044 }
1045 }
1046
1047 void compareIcons(ImageryInfo j, JsonObject e) {
1048 String ij = getIcon(j);
1049 String ie = getIcon(e);
1050 boolean ijok = isNotBlank(ij);
1051 boolean ieok = isNotBlank(ie);
1052 if (ijok && !ieok) {
1053 if (!optionNoEli) {
1054 myprintln("+ No ELI icon: "+getDescription(j));
1055 }
1056 } else if (!ijok && ieok) {
1057 myprintln("- No JOSM icon: "+getDescription(j));
1058 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1059 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1060 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1061 ij.startsWith("data:"))) {
1062 String iehttps = ie.replace("http:", "https:");
1063 if (ij.equals(iehttps)) {
1064 myprintln("+ Different icons: "+getDescription(j));
1065 } else {
1066 myprintln("* Different icons: "+getDescription(j));
1067 }
1068 }
1069 }
1070
1071 void compareCategories(ImageryInfo j, JsonObject e) {
1072 String cj = getCategory(j);
1073 String ce = getCategory(e);
1074 boolean cjok = isNotBlank(cj);
1075 boolean ceok = isNotBlank(ce);
1076 if (cjok && !ceok) {
1077 if (!optionNoEli) {
1078 myprintln("+ No ELI category: "+getDescription(j));
1079 }
1080 } else if (!cjok && ceok) {
1081 myprintln("- No JOSM category: "+getDescription(j));
1082 } else if (cjok && ceok && !Objects.equals(cj, ce)) {
1083 myprintln("* Different categories ('"+ce+"' != '"+cj+"'): "+getDescription(j));
1084 }
1085 }
1086
1087 void doMiscellaneousChecks() {
1088 myprintln("*** Miscellaneous checks: ***");
1089 Map<String, ImageryInfo> josmIds = new HashMap<>();
1090 Collection<String> all = Projections.getAllProjectionCodes();
1091 DomainValidator dv = DomainValidator.getInstance();
1092 for (String url : josmUrls.keySet()) {
1093 ImageryInfo j = josmUrls.get(url);
1094 String id = getId(j);
1095 if ("wms".equals(getType(j))) {
1096 String urlLc = url.toLowerCase(Locale.ENGLISH);
1097 if (getProjections(j).isEmpty()) {
1098 myprintln("* WMS without projections: "+getDescription(j));
1099 } else {
1100 List<String> unsupported = new LinkedList<>();
1101 List<String> old = new LinkedList<>();
1102 for (String p : getProjectionsUnstripped(j)) {
1103 if ("CRS:84".equals(p)) {
1104 if (!urlLc.contains("version=1.3")) {
1105 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
1106 }
1107 } else if (oldproj.containsKey(p)) {
1108 old.add(p);
1109 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1110 unsupported.add(p);
1111 }
1112 }
1113 if (!unsupported.isEmpty()) {
1114 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1115 }
1116 for (String o : old) {
1117 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1118 + getDescription(j));
1119 }
1120 }
1121 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1122 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1123 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1124 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
1125 }
1126 }
1127 List<String> urls = new LinkedList<>();
1128 if (!"scanex".equals(getType(j))) {
1129 urls.add(url);
1130 }
1131 String jt = getPermissionReferenceUrl(j);
1132 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1133 urls.add(jt);
1134 jt = getTermsOfUseUrl(j);
1135 if (isNotBlank(jt))
1136 urls.add(jt);
1137 jt = getAttributionUrl(j);
1138 if (isNotBlank(jt))
1139 urls.add(jt);
1140 jt = getIcon(j);
1141 if (isNotBlank(jt)) {
1142 if (!jt.startsWith("data:image/"))
1143 urls.add(jt);
1144 else {
1145 try {
1146 new ImageProvider(jt).get();
1147 } catch (RuntimeException e) {
1148 myprintln("* Strange Icon: "+getDescription(j));
1149 }
1150 }
1151 }
1152 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1153 for (String u : urls) {
1154 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1155 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1156 } else {
1157 try {
1158 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1159 String domain = jurl.getHost();
1160 int port = jurl.getPort();
1161 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1162 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1163 else if (80 == port || 443 == port) {
1164 myprintln("* Useless port '"+port+"': "+getDescription(j));
1165 }
1166 } catch (MalformedURLException e) {
1167 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1168 }
1169 }
1170 }
1171
1172 if (josmMirrors.containsKey(url)) {
1173 continue;
1174 }
1175 if (isBlank(id)) {
1176 myprintln("* No JOSM-ID: "+getDescription(j));
1177 } else if (josmIds.containsKey(id)) {
1178 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
1179 } else {
1180 josmIds.put(id, j);
1181 }
1182 String d = getDate(j);
1183 if (isNotBlank(d)) {
1184 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1185 Matcher m = patternD.matcher(d);
1186 if (!m.matches()) {
1187 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
1188 } else {
1189 try {
1190 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1191 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1192 if (second.compareTo(first) < 0) {
1193 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1194 }
1195 } catch (Exception e) {
1196 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1197 }
1198 }
1199 }
1200 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1201 myprintln("* Attribution link without text: "+getDescription(j));
1202 }
1203 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1204 myprintln("* Logo link without image: "+getDescription(j));
1205 }
1206 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1207 myprintln("* Terms of Use text without link: "+getDescription(j));
1208 }
1209 List<Shape> js = getShapes(j);
1210 if (!js.isEmpty()) {
1211 double minlat = 1000;
1212 double minlon = 1000;
1213 double maxlat = -1000;
1214 double maxlon = -1000;
1215 for (Shape s: js) {
1216 for (Coordinate p: s.getPoints()) {
1217 double lat = p.getLat();
1218 double lon = p.getLon();
1219 if (lat > maxlat) maxlat = lat;
1220 if (lon > maxlon) maxlon = lon;
1221 if (lat < minlat) minlat = lat;
1222 if (lon < minlon) minlon = lon;
1223 }
1224 }
1225 ImageryBounds b = j.getBounds();
1226 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1227 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1228 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1229 + getDescription(j));
1230 }
1231 }
1232 List<String> knownCategories = Arrays.asList(
1233 "photo", "elevation", "map", "historicmap", "osmbasedmap", "historicphoto", "qa", "other");
1234 String cat = getCategory(j);
1235 if (isBlank(cat)) {
1236 myprintln("* No category: "+getDescription(j));
1237 } else if (!knownCategories.contains(cat)) {
1238 myprintln("* Strange category "+cat+": "+getDescription(j));
1239 }
1240 }
1241 }
1242
1243 /*
1244 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1245 */
1246
1247 static String getUrl(Object e) {
1248 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1249 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1250 }
1251
1252 static String getUrlStripped(Object e) {
1253 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1254 }
1255
1256 static String getDate(Object e) {
1257 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1258 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1259 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1260 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1261 if (!start.isEmpty() && !end.isEmpty())
1262 return start+";"+end;
1263 else if (!start.isEmpty())
1264 return start+";-";
1265 else if (!end.isEmpty())
1266 return "-;"+end;
1267 return "";
1268 }
1269
1270 static Date verifyDate(String year, String month, String day) throws ParseException {
1271 String date;
1272 if (year == null) {
1273 date = "3000-01-01";
1274 } else {
1275 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1276 }
1277 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1278 df.setLenient(false);
1279 return df.parse(date);
1280 }
1281
1282 static String getId(Object e) {
1283 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1284 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1285 }
1286
1287 static String getName(Object e) {
1288 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1289 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1290 }
1291
1292 static List<ImageryInfo> getMirrors(Object e) {
1293 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1294 return Collections.emptyList();
1295 }
1296
1297 static List<String> getProjections(Object e) {
1298 List<String> r = new ArrayList<>();
1299 List<String> u = getProjectionsUnstripped(e);
1300 if (u != null) {
1301 for (String p : u) {
1302 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1303 r.add(p);
1304 }
1305 }
1306 }
1307 return r;
1308 }
1309
1310 static List<String> getProjectionsUnstripped(Object e) {
1311 List<String> r = null;
1312 if (e instanceof ImageryInfo) {
1313 r = ((ImageryInfo) e).getServerProjections();
1314 } else {
1315 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1316 if (s != null) {
1317 r = new ArrayList<>();
1318 for (JsonValue p : s.asJsonArray()) {
1319 r.add(((JsonString) p).getString());
1320 }
1321 }
1322 }
1323 return r != null ? r : Collections.emptyList();
1324 }
1325
1326 static List<Shape> getShapes(Object e) {
1327 if (e instanceof ImageryInfo) {
1328 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1329 if (bounds != null) {
1330 return bounds.getShapes();
1331 }
1332 return Collections.emptyList();
1333 }
1334 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1335 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1336 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1337 List<Shape> l = new ArrayList<>();
1338 for (JsonValue shapes: poly) {
1339 Shape s = new Shape();
1340 for (JsonValue point: shapes.asJsonArray()) {
1341 String lon = point.asJsonArray().getJsonNumber(0).toString();
1342 String lat = point.asJsonArray().getJsonNumber(1).toString();
1343 s.addPoint(lat, lon);
1344 }
1345 l.add(s);
1346 }
1347 return l;
1348 }
1349 return Collections.emptyList();
1350 }
1351
1352 static String getType(Object e) {
1353 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1354 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1355 }
1356
1357 static Integer getMinZoom(Object e) {
1358 if (e instanceof ImageryInfo) {
1359 int mz = ((ImageryInfo) e).getMinZoom();
1360 return mz == 0 ? null : mz;
1361 } else {
1362 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1363 if (num == null) return null;
1364 return num.intValue();
1365 }
1366 }
1367
1368 static Integer getMaxZoom(Object e) {
1369 if (e instanceof ImageryInfo) {
1370 int mz = ((ImageryInfo) e).getMaxZoom();
1371 return mz == 0 ? null : mz;
1372 } else {
1373 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1374 if (num == null) return null;
1375 return num.intValue();
1376 }
1377 }
1378
1379 static String getCountryCode(Object e) {
1380 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1381 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1382 }
1383
1384 static String getQuality(Object e) {
1385 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1386 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1387 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1388 }
1389
1390 static boolean getOverlay(Object e) {
1391 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1392 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1393 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1394 }
1395
1396 static String getIcon(Object e) {
1397 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1398 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1399 }
1400
1401 static String getAttributionText(Object e) {
1402 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1403 try {
1404 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1405 } catch (NullPointerException ex) {
1406 return null;
1407 }
1408 }
1409
1410 static String getAttributionUrl(Object e) {
1411 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1412 try {
1413 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1414 } catch (NullPointerException ex) {
1415 return null;
1416 }
1417 }
1418
1419 static String getTermsOfUseText(Object e) {
1420 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1421 return null;
1422 }
1423
1424 static String getTermsOfUseUrl(Object e) {
1425 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1426 return null;
1427 }
1428
1429 static String getCategory(Object e) {
1430 if (e instanceof ImageryInfo) {
1431 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1432 }
1433 return ((Map<String, JsonObject>) e).get("properties").getString("category", null);
1434 }
1435
1436 static String getLogoImage(Object e) {
1437 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1438 return null;
1439 }
1440
1441 static String getLogoUrl(Object e) {
1442 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1443 return null;
1444 }
1445
1446 static String getPermissionReferenceUrl(Object e) {
1447 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1448 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1449 }
1450
1451 static Map<String, Set<String>> getNoTileHeader(Object e) {
1452 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1453 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1454 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1455 Function.identity(),
1456 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1457 }
1458
1459 static Map<String, String> getDescriptions(Object e) {
1460 Map<String, String> res = new HashMap<>();
1461 if (e instanceof ImageryInfo) {
1462 String a = ((ImageryInfo) e).getDescription();
1463 if (a != null) res.put("en", a);
1464 } else {
1465 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1466 if (a != null) res.put("en", a.replaceAll("''", "'"));
1467 }
1468 return res;
1469 }
1470
1471 static boolean getValidGeoreference(Object e) {
1472 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1473 return false;
1474 }
1475
1476 static boolean getDefault(Object e) {
1477 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1478 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1479 }
1480
1481 String getDescription(Object o) {
1482 String url = getUrl(o);
1483 String cc = getCountryCode(o);
1484 if (cc == null) {
1485 ImageryInfo j = josmUrls.get(url);
1486 if (j != null) cc = getCountryCode(j);
1487 if (cc == null) {
1488 JsonObject e = eliUrls.get(url);
1489 if (e != null) cc = getCountryCode(e);
1490 }
1491 }
1492 if (cc == null) {
1493 cc = "";
1494 } else {
1495 cc = "["+cc+"] ";
1496 }
1497 String name = getName(o);
1498 String id = getId(o);
1499 String d = cc;
1500 if (name != null && !name.isEmpty()) {
1501 d += name;
1502 if (id != null && !id.isEmpty())
1503 d += " ["+id+"]";
1504 } else if (url != null && !url.isEmpty())
1505 d += url;
1506 if (optionShorten) {
1507 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1508 }
1509 return d;
1510 }
1511}
Note: See TracBrowser for help on using the repository browser.