source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15478

Last change on this file since 15478 was 15478, checked in by Don-vip, 4 years ago

fix #16921 - fix NPE when running ELI sync script without prior preferences file
  • Property svn:eol-style set to native
File size: 63.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2import static java.nio.charset.StandardCharsets.UTF_8;
3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
9import java.io.OutputStream;
10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
12import java.net.MalformedURLException;
13import java.net.URL;
14import java.nio.file.Files;
15import java.nio.file.Paths;
16import java.text.DecimalFormat;
17import java.text.ParseException;
18import java.text.SimpleDateFormat;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Calendar;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.Date;
25import java.util.HashMap;
26import java.util.LinkedList;
27import java.util.List;
28import java.util.Locale;
29import java.util.Map;
30import java.util.Map.Entry;
31import java.util.Objects;
32import java.util.Set;
33import java.util.function.Function;
34import java.util.regex.Matcher;
35import java.util.regex.Pattern;
36import java.util.stream.Collectors;
37
38import javax.json.Json;
39import javax.json.JsonArray;
40import javax.json.JsonNumber;
41import javax.json.JsonObject;
42import javax.json.JsonReader;
43import javax.json.JsonString;
44import javax.json.JsonValue;
45
46import org.openstreetmap.gui.jmapviewer.Coordinate;
47import org.openstreetmap.josm.data.Preferences;
48import org.openstreetmap.josm.data.imagery.ImageryInfo;
49import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
52import org.openstreetmap.josm.data.preferences.JosmUrls;
53import org.openstreetmap.josm.data.projection.Projections;
54import org.openstreetmap.josm.data.validation.routines.DomainValidator;
55import org.openstreetmap.josm.io.imagery.ImageryReader;
56import org.openstreetmap.josm.spi.preferences.Config;
57import org.openstreetmap.josm.tools.ImageProvider;
58import org.openstreetmap.josm.tools.Logging;
59import org.openstreetmap.josm.tools.OptionParser;
60import org.openstreetmap.josm.tools.OptionParser.OptionCount;
61import org.openstreetmap.josm.tools.ReflectionUtils;
62import org.xml.sax.SAXException;
63
64/**
65 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
66 * The goal is to keep both lists in sync.
67 *
68 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
69 * provides also a version in the JOSM format, but the GEOJSON is the original source
70 * format, so we read that.
71 *
72 * How to run:
73 * -----------
74 *
75 * Main JOSM binary needs to be in classpath, e.g.
76 *
77 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
78 *
79 * Add option "-h" to show the available command line flags.
80 */
81@SuppressWarnings("unchecked")
82public class SyncEditorLayerIndex {
83
84 private static final int MAXLEN = 140;
85
86 private List<ImageryInfo> josmEntries;
87 private JsonArray eliEntries;
88
89 private final Map<String, JsonObject> eliUrls = new HashMap<>();
90 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
91 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
92 private static final Map<String, String> oldproj = new HashMap<>();
93 private static final List<String> ignoreproj = new LinkedList<>();
94
95 private static String eliInputFile = "imagery_eli.geojson";
96 private static String josmInputFile = "imagery_josm.imagery.xml";
97 private static String ignoreInputFile = "imagery_josm.ignores.txt";
98 private static OutputStream outputFile;
99 private static OutputStreamWriter outputStream;
100 private static String optionOutput;
101 private static boolean optionShorten;
102 private static boolean optionNoSkip;
103 private static boolean optionXhtmlBody;
104 private static boolean optionXhtml;
105 private static String optionEliXml;
106 private static String optionJosmXml;
107 private static String optionEncoding;
108 private static boolean optionNoEli;
109 private Map<String, String> skip = new HashMap<>();
110
111 /**
112 * Main method.
113 * @param args program arguments
114 * @throws IOException if any I/O error occurs
115 * @throws ReflectiveOperationException if any reflective operation error occurs
116 * @throws SAXException if any SAX error occurs
117 */
118 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
119 Locale.setDefault(Locale.ROOT);
120 parseCommandLineArguments(args);
121 Config.setUrlsProvider(JosmUrls.getInstance());
122 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
123 Config.setPreferencesInstance(pref);
124 pref.init(false);
125 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
126 script.setupProj();
127 script.loadSkip();
128 script.start();
129 script.loadJosmEntries();
130 if (optionJosmXml != null) {
131 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
132 script.printentries(script.josmEntries, stream);
133 }
134 }
135 script.loadELIEntries();
136 if (optionEliXml != null) {
137 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
138 script.printentries(script.eliEntries, stream);
139 }
140 }
141 script.checkInOneButNotTheOther();
142 script.checkCommonEntries();
143 script.end();
144 if (outputStream != null) {
145 outputStream.close();
146 }
147 if (outputFile != null) {
148 outputFile.close();
149 }
150 }
151
152 /**
153 * Displays help on the console
154 */
155 private static void showHelp() {
156 System.out.println(getHelp());
157 System.exit(0);
158 }
159
160 static String getHelp() {
161 return "usage: java -cp build SyncEditorLayerIndex\n" +
162 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
163 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
164 "Default is imagery_eli.geojson (current directory).\n" +
165 "-h,--help show this help\n" +
166 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
167 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
168 "Default is imagery_josm.imagery.xml (current directory).\n" +
169 "-m,--noeli don't show output for ELI problems\n" +
170 "-n,--noskip don't skip known entries\n" +
171 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
172 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
173 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
174 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
175 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
176 "-X,--xhtml create XHTML for display in a web page\n";
177 }
178
179 /**
180 * Parse command line arguments.
181 * @param args program arguments
182 * @throws IOException in case of I/O error
183 */
184 static void parseCommandLineArguments(String[] args) throws IOException {
185 new OptionParser("JOSM/ELI synchronization script")
186 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
187 .addShortAlias("help", "h")
188 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
189 .addShortAlias("output", "o")
190 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
191 .addShortAlias("eli_input", "e")
192 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
193 .addShortAlias("josm_input", "j")
194 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
195 .addShortAlias("ignore_input", "i")
196 .addFlagParameter("shorten", () -> optionShorten = true)
197 .addShortAlias("shorten", "s")
198 .addFlagParameter("noskip", () -> optionNoSkip = true)
199 .addShortAlias("noskip", "n")
200 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
201 .addShortAlias("xhtmlbody", "x")
202 .addFlagParameter("xhtml", () -> optionXhtml = true)
203 .addShortAlias("xhtml", "X")
204 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
205 .addShortAlias("elixml", "p")
206 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
207 .addShortAlias("josmxml", "q")
208 .addFlagParameter("noeli", () -> optionNoEli = true)
209 .addShortAlias("noeli", "m")
210 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
211 .addShortAlias("encoding", "c")
212 .parseOptionsOrExit(Arrays.asList(args));
213
214 if (optionOutput != null && !"-".equals(optionOutput)) {
215 outputFile = Files.newOutputStream(Paths.get(optionOutput));
216 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
217 } else if (optionEncoding != null) {
218 outputStream = new OutputStreamWriter(System.out, optionEncoding);
219 }
220 }
221
222 void setupProj() {
223 oldproj.put("EPSG:3359", "EPSG:3404");
224 oldproj.put("EPSG:3785", "EPSG:3857");
225 oldproj.put("EPSG:31297", "EPGS:31287");
226 oldproj.put("EPSG:31464", "EPSG:31468");
227 oldproj.put("EPSG:54004", "EPSG:3857");
228 oldproj.put("EPSG:102100", "EPSG:3857");
229 oldproj.put("EPSG:102113", "EPSG:3857");
230 oldproj.put("EPSG:900913", "EPGS:3857");
231 ignoreproj.add("EPSG:4267");
232 ignoreproj.add("EPSG:5221");
233 ignoreproj.add("EPSG:5514");
234 ignoreproj.add("EPSG:32019");
235 ignoreproj.add("EPSG:102066");
236 ignoreproj.add("EPSG:102067");
237 ignoreproj.add("EPSG:102685");
238 ignoreproj.add("EPSG:102711");
239 }
240
241 void loadSkip() throws IOException {
242 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
243 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
244 String line;
245
246 while ((line = fr.readLine()) != null) {
247 Matcher res = pattern.matcher(line);
248 if (res.matches()) {
249 if ("Ignore".equals(res.group(1))) {
250 skip.put(res.group(2), "green");
251 } else {
252 skip.put(res.group(2), "darkgoldenrod");
253 }
254 }
255 }
256 }
257 }
258
259 void myprintlnfinal(String s) throws IOException {
260 if (outputStream != null) {
261 outputStream.write(s + System.getProperty("line.separator"));
262 } else {
263 System.out.println(s);
264 }
265 }
266
267 void myprintln(String s) throws IOException {
268 if (skip.containsKey(s)) {
269 String color = skip.get(s);
270 skip.remove(s);
271 if (optionXhtmlBody || optionXhtml) {
272 s = "<pre style=\"margin:3px;color:"+color+"\">"
273 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
274 }
275 if (!optionNoSkip) {
276 return;
277 }
278 } else if (optionXhtmlBody || optionXhtml) {
279 String color =
280 s.startsWith("***") ? "black" :
281 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
282 (s.startsWith("#") ? "indigo" :
283 (s.startsWith("!") ? "orange" : "red")));
284 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
285 }
286 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
287 return;
288 }
289 myprintlnfinal(s);
290 }
291
292 void start() throws IOException {
293 if (optionXhtml) {
294 myprintlnfinal(
295 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
296 myprintlnfinal(
297 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
298 "<title>JOSM - ELI differences</title></head><body>\n");
299 }
300 }
301
302 void end() throws IOException {
303 for (String s : skip.keySet()) {
304 myprintln("+++ Obsolete skip entry: " + s);
305 }
306 if (optionXhtml) {
307 myprintlnfinal("</body></html>\n");
308 }
309 }
310
311 void loadELIEntries() throws IOException {
312 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
313 eliEntries = jr.readObject().getJsonArray("features");
314 }
315
316 for (JsonValue e : eliEntries) {
317 String url = getUrlStripped(e);
318 if (url.contains("{z}")) {
319 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
320 url = url.replace("{z}", "{zoom}");
321 }
322 if (eliUrls.containsKey(url)) {
323 myprintln("+++ ELI-URL is not unique: "+url);
324 } else {
325 eliUrls.put(url, e.asJsonObject());
326 }
327 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
328 if (s != null) {
329 String urlLc = url.toLowerCase(Locale.ENGLISH);
330 List<String> old = new LinkedList<>();
331 for (JsonValue p : s) {
332 String proj = ((JsonString) p).getString();
333 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
334 old.add(proj);
335 }
336 }
337 if (!old.isEmpty()) {
338 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
339 }
340 }
341 }
342 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
343 }
344
345 String cdata(String s) {
346 return cdata(s, false);
347 }
348
349 String cdata(String s, boolean escape) {
350 if (escape) {
351 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
352 } else if (s.matches("[<>&]"))
353 return "<![CDATA["+s+"]]>";
354 return s;
355 }
356
357 String maininfo(Object entry, String offset) {
358 String t = getType(entry);
359 String res = offset + "<type>"+t+"</type>\n";
360 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
361 if (getMinZoom(entry) != null)
362 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
363 if (getMaxZoom(entry) != null)
364 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
365 if ("wms".equals(t)) {
366 List<String> p = getProjections(entry);
367 if (p != null) {
368 res += offset + "<projections>\n";
369 for (String c : p) {
370 res += offset + " <code>"+c+"</code>\n";
371 }
372 res += offset + "</projections>\n";
373 }
374 }
375 return res;
376 }
377
378 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
379 DecimalFormat df = new DecimalFormat("#.#######");
380 df.setRoundingMode(java.math.RoundingMode.CEILING);
381 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
382 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
383 for (Object e : entries) {
384 stream.write(" <entry"
385 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
386 + (getOverlay(e) ? " overlay=\"true\"" : "")
387 + ">\n");
388 String t;
389 if (isNotBlank(t = getName(e)))
390 stream.write(" <name>"+cdata(t, true)+"</name>\n");
391 if (isNotBlank(t = getId(e)))
392 stream.write(" <id>"+t+"</id>\n");
393 if (isNotBlank(t = getCategory(e)))
394 stream.write(" <category>"+t+"</category>\n");
395 if (isNotBlank(t = getDate(e)))
396 stream.write(" <date>"+t+"</date>\n");
397 if (isNotBlank(t = getCountryCode(e)))
398 stream.write(" <country-code>"+t+"</country-code>\n");
399 if ((getDefault(e)))
400 stream.write(" <default>true</default>\n");
401 stream.write(maininfo(e, " "));
402 if (isNotBlank(t = getAttributionText(e)))
403 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
404 if (isNotBlank(t = getAttributionUrl(e)))
405 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
406 if (isNotBlank(t = getLogoImage(e)))
407 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
408 if (isNotBlank(t = getLogoUrl(e)))
409 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
410 if (isNotBlank(t = getTermsOfUseText(e)))
411 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
412 if (isNotBlank(t = getTermsOfUseUrl(e)))
413 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
414 if (isNotBlank(t = getPermissionReferenceUrl(e)))
415 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
416 if ((getValidGeoreference(e)))
417 stream.write(" <valid-georeference>true</valid-georeference>\n");
418 if (isNotBlank(t = getIcon(e)))
419 stream.write(" <icon>"+cdata(t)+"</icon>\n");
420 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
421 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
422 }
423 for (ImageryInfo m : getMirrors(e)) {
424 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
425 }
426 double minlat = 1000;
427 double minlon = 1000;
428 double maxlat = -1000;
429 double maxlon = -1000;
430 String shapes = "";
431 String sep = "\n ";
432 try {
433 for (Shape s: getShapes(e)) {
434 shapes += " <shape>";
435 int i = 0;
436 for (Coordinate p: s.getPoints()) {
437 double lat = p.getLat();
438 double lon = p.getLon();
439 if (lat > maxlat) maxlat = lat;
440 if (lon > maxlon) maxlon = lon;
441 if (lat < minlat) minlat = lat;
442 if (lon < minlon) minlon = lon;
443 if ((i++ % 3) == 0) {
444 shapes += sep + " ";
445 }
446 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
447 }
448 shapes += sep + "</shape>\n";
449 }
450 } catch (IllegalArgumentException ignored) {
451 Logging.trace(ignored);
452 }
453 if (!shapes.isEmpty()) {
454 stream.write(" <bounds min-lat='"+df.format(minlat)
455 +"' min-lon='"+df.format(minlon)
456 +"' max-lat='"+df.format(maxlat)
457 +"' max-lon='"+df.format(maxlon)+"'>\n");
458 stream.write(shapes + " </bounds>\n");
459 }
460 stream.write(" </entry>\n");
461 }
462 stream.write("</imagery>\n");
463 stream.close();
464 }
465
466 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
467 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
468 josmEntries = reader.parse();
469 }
470
471 for (ImageryInfo e : josmEntries) {
472 if (isBlank(getUrl(e))) {
473 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
474 continue;
475 }
476 if (isBlank(getName(e))) {
477 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
478 continue;
479 }
480 String url = getUrlStripped(e);
481 if (url.contains("{z}")) {
482 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
483 url = url.replace("{z}", "{zoom}");
484 }
485 if (josmUrls.containsKey(url)) {
486 myprintln("+++ JOSM-URL is not unique: "+url);
487 } else {
488 josmUrls.put(url, e);
489 }
490 for (ImageryInfo m : e.getMirrors()) {
491 url = getUrlStripped(m);
492 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
493 ReflectionUtils.setObjectsAccessible(origNameField);
494 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
495 if (josmUrls.containsKey(url)) {
496 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
497 } else {
498 josmUrls.put(url, m);
499 josmMirrors.put(url, m);
500 }
501 }
502 }
503 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
504 }
505
506 void checkInOneButNotTheOther() throws IOException {
507 List<String> le = new LinkedList<>(eliUrls.keySet());
508 List<String> lj = new LinkedList<>(josmUrls.keySet());
509
510 List<String> ke = new LinkedList<>(le);
511 for (String url : ke) {
512 if (lj.contains(url)) {
513 le.remove(url);
514 lj.remove(url);
515 }
516 }
517
518 if (!le.isEmpty() && !lj.isEmpty()) {
519 ke = new LinkedList<>(le);
520 for (String urle : ke) {
521 JsonObject e = eliUrls.get(urle);
522 String ide = getId(e);
523 String urlhttps = urle.replace("http:", "https:");
524 if (lj.contains(urlhttps)) {
525 myprintln("+ Missing https: "+getDescription(e));
526 eliUrls.put(urlhttps, eliUrls.get(urle));
527 eliUrls.remove(urle);
528 le.remove(urle);
529 lj.remove(urlhttps);
530 } else if (isNotBlank(ide)) {
531 List<String> kj = new LinkedList<>(lj);
532 for (String urlj : kj) {
533 ImageryInfo j = josmUrls.get(urlj);
534 String idj = getId(j);
535
536 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
537 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
538 le.remove(urle);
539 lj.remove(urlj);
540 // replace key for this entry with JOSM URL
541 eliUrls.remove(urle);
542 eliUrls.put(urlj, e);
543 break;
544 }
545 }
546 }
547 }
548 }
549
550 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
551 Collections.sort(le);
552 if (!le.isEmpty()) {
553 for (String l : le) {
554 myprintln("- " + getDescription(eliUrls.get(l)));
555 }
556 }
557 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
558 Collections.sort(lj);
559 if (!lj.isEmpty()) {
560 for (String l : lj) {
561 myprintln("+ " + getDescription(josmUrls.get(l)));
562 }
563 }
564 }
565
566 void checkCommonEntries() throws IOException {
567 doSameUrlButDifferentName();
568 doSameUrlButDifferentId();
569 doSameUrlButDifferentType();
570 doSameUrlButDifferentZoomBounds();
571 doSameUrlButDifferentCountryCode();
572 doSameUrlButDifferentQuality();
573 doSameUrlButDifferentDates();
574 doSameUrlButDifferentInformation();
575 doMismatchingShapes();
576 doMismatchingIcons();
577 doMiscellaneousChecks();
578 }
579
580 void doSameUrlButDifferentName() throws IOException {
581 myprintln("*** Same URL, but different name: ***");
582 for (String url : eliUrls.keySet()) {
583 JsonObject e = eliUrls.get(url);
584 if (!josmUrls.containsKey(url)) continue;
585 ImageryInfo j = josmUrls.get(url);
586 String ename = getName(e).replace("'", "\u2019");
587 String jname = getName(j).replace("'", "\u2019");
588 if (!ename.equals(jname)) {
589 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
590 }
591 }
592 }
593
594 void doSameUrlButDifferentId() throws IOException {
595 myprintln("*** Same URL, but different Id: ***");
596 for (String url : eliUrls.keySet()) {
597 JsonObject e = eliUrls.get(url);
598 if (!josmUrls.containsKey(url)) continue;
599 ImageryInfo j = josmUrls.get(url);
600 String ename = getId(e);
601 String jname = getId(j);
602 if (!Objects.equals(ename, jname)) {
603 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
604 }
605 }
606 }
607
608 void doSameUrlButDifferentType() throws IOException {
609 myprintln("*** Same URL, but different type: ***");
610 for (String url : eliUrls.keySet()) {
611 JsonObject e = eliUrls.get(url);
612 if (!josmUrls.containsKey(url)) continue;
613 ImageryInfo j = josmUrls.get(url);
614 if (!Objects.equals(getType(e), getType(j))) {
615 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
616 }
617 }
618 }
619
620 void doSameUrlButDifferentZoomBounds() throws IOException {
621 myprintln("*** Same URL, but different zoom bounds: ***");
622 for (String url : eliUrls.keySet()) {
623 JsonObject e = eliUrls.get(url);
624 if (!josmUrls.containsKey(url)) continue;
625 ImageryInfo j = josmUrls.get(url);
626
627 Integer eMinZoom = getMinZoom(e);
628 Integer jMinZoom = getMinZoom(j);
629 /* dont warn for entries copied from the base of the mirror */
630 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
631 jMinZoom = null;
632 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
633 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
634 }
635 Integer eMaxZoom = getMaxZoom(e);
636 Integer jMaxZoom = getMaxZoom(j);
637 /* dont warn for entries copied from the base of the mirror */
638 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
639 jMaxZoom = null;
640 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
641 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
642 }
643 }
644 }
645
646 void doSameUrlButDifferentCountryCode() throws IOException {
647 myprintln("*** Same URL, but different country code: ***");
648 for (String url : eliUrls.keySet()) {
649 JsonObject e = eliUrls.get(url);
650 if (!josmUrls.containsKey(url)) continue;
651 ImageryInfo j = josmUrls.get(url);
652 String cce = getCountryCode(e);
653 if ("ZZ".equals(cce)) { /* special ELI country code */
654 cce = null;
655 }
656 if (cce != null && !cce.equals(getCountryCode(j))) {
657 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
658 }
659 }
660 }
661
662 void doSameUrlButDifferentQuality() throws IOException {
663 myprintln("*** Same URL, but different quality: ***");
664 for (String url : eliUrls.keySet()) {
665 JsonObject e = eliUrls.get(url);
666 if (!josmUrls.containsKey(url)) {
667 String q = getQuality(e);
668 if ("eli-best".equals(q)) {
669 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
670 }
671 continue;
672 }
673 ImageryInfo j = josmUrls.get(url);
674 if (!Objects.equals(getQuality(e), getQuality(j))) {
675 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
676 }
677 }
678 }
679
680 void doSameUrlButDifferentDates() throws IOException {
681 myprintln("*** Same URL, but different dates: ***");
682 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
683 for (String url : eliUrls.keySet()) {
684 String ed = getDate(eliUrls.get(url));
685 if (!josmUrls.containsKey(url)) continue;
686 ImageryInfo j = josmUrls.get(url);
687 String jd = getDate(j);
688 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
689 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
690 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
691 String ed2 = ed;
692 Matcher m = pattern.matcher(ed);
693 if (m.matches()) {
694 Calendar cal = Calendar.getInstance();
695 cal.set(Integer.valueOf(m.group(2)),
696 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
697 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
698 cal.add(Calendar.DAY_OF_MONTH, -1);
699 ed2 = m.group(1) + cal.get(Calendar.YEAR);
700 if (m.group(4) != null)
701 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
702 if (m.group(6) != null)
703 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
704 }
705 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
706 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
707 String t = "'"+ed+"'";
708 if (!ed.equals(ef)) {
709 t += " or '"+ef+"'";
710 }
711 if (jd.isEmpty()) {
712 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
713 } else if (!ed.isEmpty()) {
714 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
715 } else if (!optionNoEli) {
716 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
717 }
718 }
719 }
720 }
721
722 void doSameUrlButDifferentInformation() throws IOException {
723 myprintln("*** Same URL, but different information: ***");
724 for (String url : eliUrls.keySet()) {
725 if (!josmUrls.containsKey(url)) continue;
726 JsonObject e = eliUrls.get(url);
727 ImageryInfo j = josmUrls.get(url);
728
729 compareDescriptions(e, j);
730 comparePermissionReferenceUrls(e, j);
731 compareAttributionUrls(e, j);
732 compareAttributionTexts(e, j);
733 compareProjections(e, j);
734 compareDefaults(e, j);
735 compareOverlays(e, j);
736 compareNoTileHeaders(e, j);
737 }
738 }
739
740 void compareDescriptions(JsonObject e, ImageryInfo j) throws IOException {
741 String et = getDescriptions(e).getOrDefault("en", "");
742 String jt = getDescriptions(j).getOrDefault("en", "");
743 if (!et.equals(jt)) {
744 if (jt.isEmpty()) {
745 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
746 } else if (!et.isEmpty()) {
747 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
748 } else if (!optionNoEli) {
749 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
750 }
751 }
752 }
753
754 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) throws IOException {
755 String et = getPermissionReferenceUrl(e);
756 String jt = getPermissionReferenceUrl(j);
757 String jt2 = getTermsOfUseUrl(j);
758 if (isBlank(jt)) jt = jt2;
759 if (!Objects.equals(et, jt)) {
760 if (isBlank(jt)) {
761 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
762 } else if (isNotBlank(et)) {
763 String ethttps = et.replace("http:", "https:");
764 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
765 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
766 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
767 } else {
768 String ja = getAttributionUrl(j);
769 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
770 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
771 } else {
772 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
773 }
774 }
775 }
776 } else if (!optionNoEli) {
777 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
778 }
779 }
780 }
781
782 void compareAttributionUrls(JsonObject e, ImageryInfo j) throws IOException {
783 String et = getAttributionUrl(e);
784 String jt = getAttributionUrl(j);
785 if (!Objects.equals(et, jt)) {
786 if (isBlank(jt)) {
787 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
788 } else if (isNotBlank(et)) {
789 String ethttps = et.replace("http:", "https:");
790 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
791 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
792 } else {
793 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
794 }
795 } else if (!optionNoEli) {
796 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
797 }
798 }
799 }
800
801 void compareAttributionTexts(JsonObject e, ImageryInfo j) throws IOException {
802 String et = getAttributionText(e);
803 String jt = getAttributionText(j);
804 if (!Objects.equals(et, jt)) {
805 if (isBlank(jt)) {
806 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
807 } else if (isNotBlank(et)) {
808 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
809 } else if (!optionNoEli) {
810 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
811 }
812 }
813 }
814
815 void compareProjections(JsonObject e, ImageryInfo j) throws IOException {
816 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
817 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
818 if (!Objects.equals(et, jt)) {
819 if (isBlank(jt)) {
820 String t = getType(e);
821 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
822 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
823 } else {
824 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
825 }
826 } else if (isNotBlank(et)) {
827 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
828 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
829 } else {
830 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
831 }
832 } else if (!optionNoEli && !"tms".equals(getType(e))) {
833 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
834 }
835 }
836 }
837
838 void compareDefaults(JsonObject e, ImageryInfo j) throws IOException {
839 boolean ed = getDefault(e);
840 boolean jd = getDefault(j);
841 if (ed != jd) {
842 if (!jd) {
843 myprintln("- Missing JOSM default: "+getDescription(j));
844 } else if (!optionNoEli) {
845 myprintln("+ Missing ELI default: "+getDescription(j));
846 }
847 }
848 }
849
850 void compareOverlays(JsonObject e, ImageryInfo j) throws IOException {
851 boolean eo = getOverlay(e);
852 boolean jo = getOverlay(j);
853 if (eo != jo) {
854 if (!jo) {
855 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
856 } else if (!optionNoEli) {
857 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
858 }
859 }
860 }
861
862 void compareNoTileHeaders(JsonObject e, ImageryInfo j) throws IOException {
863 Map<String, Set<String>> eh = getNoTileHeader(e);
864 Map<String, Set<String>> jh = getNoTileHeader(j);
865 if (!Objects.equals(eh, jh)) {
866 if (jh == null || jh.isEmpty()) {
867 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
868 } else if (eh != null && !eh.isEmpty()) {
869 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
870 } else if (!optionNoEli) {
871 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
872 }
873 }
874 }
875
876 void doMismatchingShapes() throws IOException {
877 myprintln("*** Mismatching shapes: ***");
878 for (String url : josmUrls.keySet()) {
879 ImageryInfo j = josmUrls.get(url);
880 int num = 1;
881 for (Shape shape : getShapes(j)) {
882 List<Coordinate> p = shape.getPoints();
883 if (!p.get(0).equals(p.get(p.size()-1))) {
884 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
885 }
886 for (int nump = 1; nump < p.size(); ++nump) {
887 if (Objects.equals(p.get(nump-1), p.get(nump))) {
888 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
889 }
890 }
891 ++num;
892 }
893 }
894 for (String url : eliUrls.keySet()) {
895 JsonObject e = eliUrls.get(url);
896 int num = 1;
897 List<Shape> s = null;
898 try {
899 s = getShapes(e);
900 for (Shape shape : s) {
901 List<Coordinate> p = shape.getPoints();
902 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
903 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
904 }
905 for (int nump = 1; nump < p.size(); ++nump) {
906 if (Objects.equals(p.get(nump-1), p.get(nump))) {
907 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
908 }
909 }
910 ++num;
911 }
912 } catch (IllegalArgumentException err) {
913 String desc = getDescription(e);
914 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
915 }
916 if (s == null || !josmUrls.containsKey(url)) {
917 continue;
918 }
919 ImageryInfo j = josmUrls.get(url);
920 List<Shape> js = getShapes(j);
921 if (s.isEmpty() && !js.isEmpty()) {
922 if (!optionNoEli) {
923 myprintln("+ No ELI shape: "+getDescription(j));
924 }
925 } else if (js.isEmpty() && !s.isEmpty()) {
926 // don't report boundary like 5 point shapes as difference
927 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
928 myprintln("- No JOSM shape: "+getDescription(j));
929 }
930 } else if (s.size() != js.size()) {
931 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
932 } else {
933 boolean[] edone = new boolean[s.size()];
934 boolean[] jdone = new boolean[js.size()];
935 for (int enums = 0; enums < s.size(); ++enums) {
936 List<Coordinate> ep = s.get(enums).getPoints();
937 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
938 List<Coordinate> jp = js.get(jnums).getPoints();
939 if (ep.size() == jp.size() && !jdone[jnums]) {
940 boolean err = false;
941 for (int nump = 0; nump < ep.size() && !err; ++nump) {
942 Coordinate ept = ep.get(nump);
943 Coordinate jpt = jp.get(nump);
944 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
945 err = true;
946 }
947 if (!err) {
948 edone[enums] = true;
949 jdone[jnums] = true;
950 break;
951 }
952 }
953 }
954 }
955 for (int enums = 0; enums < s.size(); ++enums) {
956 List<Coordinate> ep = s.get(enums).getPoints();
957 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
958 List<Coordinate> jp = js.get(jnums).getPoints();
959 if (ep.size() == jp.size() && !jdone[jnums]) {
960 boolean err = false;
961 for (int nump = 0; nump < ep.size() && !err; ++nump) {
962 Coordinate ept = ep.get(nump);
963 Coordinate jpt = jp.get(nump);
964 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
965 String numtxt = Integer.toString(enums+1);
966 if (enums != jnums) {
967 numtxt += '/' + Integer.toString(jnums+1);
968 }
969 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
970 break;
971 }
972 }
973 edone[enums] = true;
974 jdone[jnums] = true;
975 break;
976 }
977 }
978 }
979 for (int enums = 0; enums < s.size(); ++enums) {
980 List<Coordinate> ep = s.get(enums).getPoints();
981 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
982 List<Coordinate> jp = js.get(jnums).getPoints();
983 if (!jdone[jnums]) {
984 String numtxt = Integer.toString(enums+1);
985 if (enums != jnums) {
986 numtxt += '/' + Integer.toString(jnums+1);
987 }
988 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
989 + getDescription(j));
990 edone[enums] = true;
991 jdone[jnums] = true;
992 break;
993 }
994 }
995 }
996 }
997 }
998 }
999
1000 void doMismatchingIcons() throws IOException {
1001 myprintln("*** Mismatching icons: ***");
1002 for (String url : eliUrls.keySet()) {
1003 JsonObject e = eliUrls.get(url);
1004 if (!josmUrls.containsKey(url)) {
1005 continue;
1006 }
1007 ImageryInfo j = josmUrls.get(url);
1008 String ij = getIcon(j);
1009 String ie = getIcon(e);
1010 boolean ijok = isNotBlank(ij);
1011 boolean ieok = isNotBlank(ie);
1012 if (ijok && !ieok) {
1013 if (!optionNoEli) {
1014 myprintln("+ No ELI icon: "+getDescription(j));
1015 }
1016 } else if (!ijok && ieok) {
1017 myprintln("- No JOSM icon: "+getDescription(j));
1018 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
1019 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1020 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
1021 ij.startsWith("data:"))) {
1022 String iehttps = ie.replace("http:", "https:");
1023 if (ij.equals(iehttps)) {
1024 myprintln("+ Different icons: "+getDescription(j));
1025 } else {
1026 myprintln("* Different icons: "+getDescription(j));
1027 }
1028 }
1029 }
1030 }
1031
1032 void doMiscellaneousChecks() throws IOException {
1033 myprintln("*** Miscellaneous checks: ***");
1034 Map<String, ImageryInfo> josmIds = new HashMap<>();
1035 Collection<String> all = Projections.getAllProjectionCodes();
1036 DomainValidator dv = DomainValidator.getInstance();
1037 for (String url : josmUrls.keySet()) {
1038 ImageryInfo j = josmUrls.get(url);
1039 String id = getId(j);
1040 if ("wms".equals(getType(j))) {
1041 String urlLc = url.toLowerCase(Locale.ENGLISH);
1042 if (getProjections(j).isEmpty()) {
1043 myprintln("* WMS without projections: "+getDescription(j));
1044 } else {
1045 List<String> unsupported = new LinkedList<>();
1046 List<String> old = new LinkedList<>();
1047 for (String p : getProjectionsUnstripped(j)) {
1048 if ("CRS:84".equals(p)) {
1049 if (!urlLc.contains("version=1.3")) {
1050 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
1051 }
1052 } else if (oldproj.containsKey(p)) {
1053 old.add(p);
1054 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1055 unsupported.add(p);
1056 }
1057 }
1058 if (!unsupported.isEmpty()) {
1059 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
1060 }
1061 for (String o : old) {
1062 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1063 + getDescription(j));
1064 }
1065 }
1066 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1067 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1068 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1069 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
1070 }
1071 }
1072 List<String> urls = new LinkedList<>();
1073 if (!"scanex".equals(getType(j))) {
1074 urls.add(url);
1075 }
1076 String jt = getPermissionReferenceUrl(j);
1077 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1078 urls.add(jt);
1079 jt = getTermsOfUseUrl(j);
1080 if (isNotBlank(jt))
1081 urls.add(jt);
1082 jt = getAttributionUrl(j);
1083 if (isNotBlank(jt))
1084 urls.add(jt);
1085 jt = getIcon(j);
1086 if (isNotBlank(jt)) {
1087 if (!jt.startsWith("data:image/"))
1088 urls.add(jt);
1089 else {
1090 try {
1091 new ImageProvider(jt).get();
1092 } catch (RuntimeException e) {
1093 myprintln("* Strange Icon: "+getDescription(j));
1094 }
1095 }
1096 }
1097 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
1098 for (String u : urls) {
1099 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
1100 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1101 } else {
1102 try {
1103 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1104 String domain = jurl.getHost();
1105 int port = jurl.getPort();
1106 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1107 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1108 else if (80 == port || 443 == port) {
1109 myprintln("* Useless port '"+port+"': "+getDescription(j));
1110 }
1111 } catch (MalformedURLException e) {
1112 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
1113 }
1114 }
1115 }
1116
1117 if (josmMirrors.containsKey(url)) {
1118 continue;
1119 }
1120 if (isBlank(id)) {
1121 myprintln("* No JOSM-ID: "+getDescription(j));
1122 } else if (josmIds.containsKey(id)) {
1123 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
1124 } else {
1125 josmIds.put(id, j);
1126 }
1127 String d = getDate(j);
1128 if (isNotBlank(d)) {
1129 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1130 Matcher m = patternD.matcher(d);
1131 if (!m.matches()) {
1132 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
1133 } else {
1134 try {
1135 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1136 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1137 if (second.compareTo(first) < 0) {
1138 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
1139 }
1140 } catch (Exception e) {
1141 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
1142 }
1143 }
1144 }
1145 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1146 myprintln("* Attribution link without text: "+getDescription(j));
1147 }
1148 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1149 myprintln("* Logo link without image: "+getDescription(j));
1150 }
1151 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1152 myprintln("* Terms of Use text without link: "+getDescription(j));
1153 }
1154 List<Shape> js = getShapes(j);
1155 if (!js.isEmpty()) {
1156 double minlat = 1000;
1157 double minlon = 1000;
1158 double maxlat = -1000;
1159 double maxlon = -1000;
1160 for (Shape s: js) {
1161 for (Coordinate p: s.getPoints()) {
1162 double lat = p.getLat();
1163 double lon = p.getLon();
1164 if (lat > maxlat) maxlat = lat;
1165 if (lon > maxlon) maxlon = lon;
1166 if (lat < minlat) minlat = lat;
1167 if (lon < minlon) minlon = lon;
1168 }
1169 }
1170 ImageryBounds b = j.getBounds();
1171 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1172 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
1173 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1174 + getDescription(j));
1175 }
1176 }
1177 List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
1178 String cat = getCategory(j);
1179 if (isBlank(cat)) {
1180 myprintln("* No category: "+getDescription(j));
1181 } else if (!knownCategories.contains(cat)) {
1182 myprintln("* Strange category "+cat+": "+getDescription(j));
1183 }
1184 }
1185 }
1186
1187 /*
1188 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1189 */
1190
1191 static String getUrl(Object e) {
1192 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1193 return ((Map<String, JsonObject>) e).get("properties").getString("url");
1194 }
1195
1196 static String getUrlStripped(Object e) {
1197 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
1198 }
1199
1200 static String getDate(Object e) {
1201 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1202 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1203 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1204 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
1205 if (!start.isEmpty() && !end.isEmpty())
1206 return start+";"+end;
1207 else if (!start.isEmpty())
1208 return start+";-";
1209 else if (!end.isEmpty())
1210 return "-;"+end;
1211 return "";
1212 }
1213
1214 static Date verifyDate(String year, String month, String day) throws ParseException {
1215 String date;
1216 if (year == null) {
1217 date = "3000-01-01";
1218 } else {
1219 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
1220 }
1221 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1222 df.setLenient(false);
1223 return df.parse(date);
1224 }
1225
1226 static String getId(Object e) {
1227 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1228 return ((Map<String, JsonObject>) e).get("properties").getString("id");
1229 }
1230
1231 static String getName(Object e) {
1232 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1233 return ((Map<String, JsonObject>) e).get("properties").getString("name");
1234 }
1235
1236 static List<ImageryInfo> getMirrors(Object e) {
1237 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1238 return Collections.emptyList();
1239 }
1240
1241 static List<String> getProjections(Object e) {
1242 List<String> r = new ArrayList<>();
1243 List<String> u = getProjectionsUnstripped(e);
1244 if (u != null) {
1245 for (String p : u) {
1246 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1247 r.add(p);
1248 }
1249 }
1250 }
1251 return r;
1252 }
1253
1254 static List<String> getProjectionsUnstripped(Object e) {
1255 List<String> r = null;
1256 if (e instanceof ImageryInfo) {
1257 r = ((ImageryInfo) e).getServerProjections();
1258 } else {
1259 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1260 if (s != null) {
1261 r = new ArrayList<>();
1262 for (JsonValue p : s.asJsonArray()) {
1263 r.add(((JsonString) p).getString());
1264 }
1265 }
1266 }
1267 return r != null ? r : Collections.emptyList();
1268 }
1269
1270 static List<Shape> getShapes(Object e) {
1271 if (e instanceof ImageryInfo) {
1272 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
1273 if (bounds != null) {
1274 return bounds.getShapes();
1275 }
1276 return Collections.emptyList();
1277 }
1278 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1279 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1280 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1281 List<Shape> l = new ArrayList<>();
1282 for (JsonValue shapes: poly) {
1283 Shape s = new Shape();
1284 for (JsonValue point: shapes.asJsonArray()) {
1285 String lon = point.asJsonArray().getJsonNumber(0).toString();
1286 String lat = point.asJsonArray().getJsonNumber(1).toString();
1287 s.addPoint(lat, lon);
1288 }
1289 l.add(s);
1290 }
1291 return l;
1292 }
1293 return Collections.emptyList();
1294 }
1295
1296 static String getType(Object e) {
1297 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1298 return ((Map<String, JsonObject>) e).get("properties").getString("type");
1299 }
1300
1301 static Integer getMinZoom(Object e) {
1302 if (e instanceof ImageryInfo) {
1303 int mz = ((ImageryInfo) e).getMinZoom();
1304 return mz == 0 ? null : mz;
1305 } else {
1306 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1307 if (num == null) return null;
1308 return num.intValue();
1309 }
1310 }
1311
1312 static Integer getMaxZoom(Object e) {
1313 if (e instanceof ImageryInfo) {
1314 int mz = ((ImageryInfo) e).getMaxZoom();
1315 return mz == 0 ? null : mz;
1316 } else {
1317 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1318 if (num == null) return null;
1319 return num.intValue();
1320 }
1321 }
1322
1323 static String getCountryCode(Object e) {
1324 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1325 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
1326 }
1327
1328 static String getQuality(Object e) {
1329 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1330 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1331 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
1332 }
1333
1334 static boolean getOverlay(Object e) {
1335 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1336 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1337 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
1338 }
1339
1340 static String getIcon(Object e) {
1341 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1342 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
1343 }
1344
1345 static String getAttributionText(Object e) {
1346 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1347 try {
1348 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1349 } catch (NullPointerException ex) {
1350 return null;
1351 }
1352 }
1353
1354 static String getAttributionUrl(Object e) {
1355 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1356 try {
1357 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1358 } catch (NullPointerException ex) {
1359 return null;
1360 }
1361 }
1362
1363 static String getTermsOfUseText(Object e) {
1364 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1365 return null;
1366 }
1367
1368 static String getTermsOfUseUrl(Object e) {
1369 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1370 return null;
1371 }
1372
1373 static String getCategory(Object e) {
1374 if (e instanceof ImageryInfo) {
1375 return ((ImageryInfo) e).getImageryCategoryOriginalString();
1376 }
1377 return null;
1378 }
1379
1380 static String getLogoImage(Object e) {
1381 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1382 return null;
1383 }
1384
1385 static String getLogoUrl(Object e) {
1386 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1387 return null;
1388 }
1389
1390 static String getPermissionReferenceUrl(Object e) {
1391 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1392 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
1393 }
1394
1395 static Map<String, Set<String>> getNoTileHeader(Object e) {
1396 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1397 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1398 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1399 Function.identity(),
1400 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1401 }
1402
1403 static Map<String, String> getDescriptions(Object e) {
1404 Map<String, String> res = new HashMap<>();
1405 if (e instanceof ImageryInfo) {
1406 String a = ((ImageryInfo) e).getDescription();
1407 if (a != null) res.put("en", a);
1408 } else {
1409 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
1410 if (a != null) res.put("en", a.replaceAll("''", "'"));
1411 }
1412 return res;
1413 }
1414
1415 static boolean getValidGeoreference(Object e) {
1416 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1417 return false;
1418 }
1419
1420 static boolean getDefault(Object e) {
1421 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1422 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
1423 }
1424
1425 String getDescription(Object o) {
1426 String url = getUrl(o);
1427 String cc = getCountryCode(o);
1428 if (cc == null) {
1429 ImageryInfo j = josmUrls.get(url);
1430 if (j != null) cc = getCountryCode(j);
1431 if (cc == null) {
1432 JsonObject e = eliUrls.get(url);
1433 if (e != null) cc = getCountryCode(e);
1434 }
1435 }
1436 if (cc == null) {
1437 cc = "";
1438 } else {
1439 cc = "["+cc+"] ";
1440 }
1441 String d = cc + getName(o) + " - " + getUrl(o);
1442 if (optionShorten) {
1443 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
1444 }
1445 return d;
1446 }
1447}
Note: See TracBrowser for help on using the repository browser.