source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15439

Last change on this file since 15439 was 15439, checked in by stoecker, 6 years ago

add check for broken base64 encoded icons

  • Property svn:eol-style set to native
File size: 63.3 KB
RevLine 
[7726]1// License: GPL. For details, see LICENSE file.
[15034]2import static java.nio.charset.StandardCharsets.UTF_8;
[15033]3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
[15034]9import java.io.OutputStream;
[15033]10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
[15163]12import java.net.MalformedURLException;
13import java.net.URL;
[15034]14import java.nio.file.Files;
15import java.nio.file.Paths;
[15033]16import java.text.DecimalFormat;
17import java.text.ParseException;
18import java.text.SimpleDateFormat;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Calendar;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.Date;
25import java.util.HashMap;
26import java.util.LinkedList;
27import java.util.List;
28import java.util.Locale;
29import java.util.Map;
30import java.util.Map.Entry;
31import java.util.Objects;
[15082]32import java.util.Set;
33import java.util.function.Function;
[15033]34import java.util.regex.Matcher;
35import java.util.regex.Pattern;
36import java.util.stream.Collectors;
37
38import javax.json.Json;
39import javax.json.JsonArray;
40import javax.json.JsonNumber;
41import javax.json.JsonObject;
42import javax.json.JsonReader;
43import javax.json.JsonString;
44import javax.json.JsonValue;
45
46import org.openstreetmap.gui.jmapviewer.Coordinate;
47import org.openstreetmap.josm.data.Preferences;
48import org.openstreetmap.josm.data.imagery.ImageryInfo;
49import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
52import org.openstreetmap.josm.data.projection.Projections;
53import org.openstreetmap.josm.data.validation.routines.DomainValidator;
54import org.openstreetmap.josm.io.imagery.ImageryReader;
55import org.openstreetmap.josm.spi.preferences.Config;
[15439]56import org.openstreetmap.josm.tools.ImageProvider;
[15034]57import org.openstreetmap.josm.tools.Logging;
[15033]58import org.openstreetmap.josm.tools.OptionParser;
59import org.openstreetmap.josm.tools.OptionParser.OptionCount;
60import org.openstreetmap.josm.tools.ReflectionUtils;
61import org.xml.sax.SAXException;
62
[7726]63/**
[11854]64 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
[7726]65 * The goal is to keep both lists in sync.
66 *
[11854]67 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
[11694]68 * provides also a version in the JOSM format, but the GEOJSON is the original source
[7726]69 * format, so we read that.
70 *
71 * How to run:
72 * -----------
73 *
74 * Main JOSM binary needs to be in classpath, e.g.
75 *
[15033]76 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
[9667]77 *
[7726]78 * Add option "-h" to show the available command line flags.
79 */
[15037]80@SuppressWarnings("unchecked")
[15033]81public class SyncEditorLayerIndex {
[14019]82
[15034]83 private static final int MAXLEN = 140;
84
[15033]85 private List<ImageryInfo> josmEntries;
86 private JsonArray eliEntries;
[7726]87
[15034]88 private final Map<String, JsonObject> eliUrls = new HashMap<>();
89 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
90 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
91 private static final Map<String, String> oldproj = new HashMap<>();
92 private static final List<String> ignoreproj = new LinkedList<>();
[7726]93
[15033]94 private static String eliInputFile = "imagery_eli.geojson";
95 private static String josmInputFile = "imagery_josm.imagery.xml";
96 private static String ignoreInputFile = "imagery_josm.ignores.txt";
[15034]97 private static OutputStream outputFile;
98 private static OutputStreamWriter outputStream;
[15033]99 private static String optionOutput;
100 private static boolean optionShorten;
101 private static boolean optionNoSkip;
102 private static boolean optionXhtmlBody;
103 private static boolean optionXhtml;
104 private static String optionEliXml;
105 private static String optionJosmXml;
106 private static String optionEncoding;
107 private static boolean optionNoEli;
108 private Map<String, String> skip = new HashMap<>();
[7726]109
110 /**
111 * Main method.
[15033]112 * @param args program arguments
113 * @throws IOException if any I/O error occurs
114 * @throws ReflectiveOperationException if any reflective operation error occurs
115 * @throws SAXException if any SAX error occurs
[7726]116 */
[15033]117 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
118 Locale.setDefault(Locale.ROOT);
[15034]119 parseCommandLineArguments(args);
[15033]120 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
121 Config.setPreferencesInstance(pref);
122 pref.init(false);
123 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
124 script.setupProj();
125 script.loadSkip();
126 script.start();
127 script.loadJosmEntries();
128 if (optionJosmXml != null) {
[15034]129 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
130 script.printentries(script.josmEntries, stream);
131 }
[11964]132 }
[15033]133 script.loadELIEntries();
134 if (optionEliXml != null) {
[15034]135 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
136 script.printentries(script.eliEntries, stream);
137 }
[11964]138 }
[15033]139 script.checkInOneButNotTheOther();
140 script.checkCommonEntries();
141 script.end();
142 if (outputStream != null) {
143 outputStream.close();
[9505]144 }
[15033]145 if (outputFile != null) {
146 outputFile.close();
[9505]147 }
[7726]148 }
[9653]149
[7726]150 /**
[15033]151 * Displays help on the console
152 */
153 private static void showHelp() {
154 System.out.println(getHelp());
155 System.exit(0);
156 }
157
158 static String getHelp() {
159 return "usage: java -cp build SyncEditorLayerIndex\n" +
160 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
[15034]161 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
162 "Default is imagery_eli.geojson (current directory).\n" +
[15033]163 "-h,--help show this help\n" +
164 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
[15034]165 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
166 "Default is imagery_josm.imagery.xml (current directory).\n" +
[15033]167 "-m,--noeli don't show output for ELI problems\n" +
168 "-n,--noskip don't skip known entries\n" +
169 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
170 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
171 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
172 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
173 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
174 "-X,--xhtml create XHTML for display in a web page\n";
175 }
176
177 /**
[7726]178 * Parse command line arguments.
[15033]179 * @param args program arguments
[15034]180 * @throws IOException in case of I/O error
[7726]181 */
[15034]182 static void parseCommandLineArguments(String[] args) throws IOException {
[15033]183 new OptionParser("JOSM/ELI synchronization script")
184 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
185 .addShortAlias("help", "h")
186 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
187 .addShortAlias("output", "o")
188 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
189 .addShortAlias("eli_input", "e")
190 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
191 .addShortAlias("josm_input", "j")
192 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
193 .addShortAlias("ignore_input", "i")
194 .addFlagParameter("shorten", () -> optionShorten = true)
195 .addShortAlias("shorten", "s")
196 .addFlagParameter("noskip", () -> optionNoSkip = true)
197 .addShortAlias("noskip", "n")
198 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
199 .addShortAlias("xhtmlbody", "x")
200 .addFlagParameter("xhtml", () -> optionXhtml = true)
201 .addShortAlias("xhtml", "X")
202 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
203 .addShortAlias("elixml", "p")
204 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
205 .addShortAlias("josmxml", "q")
206 .addFlagParameter("noeli", () -> optionNoEli = true)
207 .addShortAlias("noeli", "m")
208 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
209 .addShortAlias("encoding", "c")
210 .parseOptionsOrExit(Arrays.asList(args));
[7726]211
[15034]212 if (optionOutput != null && !"-".equals(optionOutput)) {
213 outputFile = Files.newOutputStream(Paths.get(optionOutput));
[15033]214 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
215 } else if (optionEncoding != null) {
216 outputStream = new OutputStreamWriter(System.out, optionEncoding);
[7726]217 }
218 }
219
[13530]220 void setupProj() {
[15033]221 oldproj.put("EPSG:3359", "EPSG:3404");
222 oldproj.put("EPSG:3785", "EPSG:3857");
223 oldproj.put("EPSG:31297", "EPGS:31287");
224 oldproj.put("EPSG:31464", "EPSG:31468");
225 oldproj.put("EPSG:54004", "EPSG:3857");
226 oldproj.put("EPSG:102100", "EPSG:3857");
227 oldproj.put("EPSG:102113", "EPSG:3857");
228 oldproj.put("EPSG:900913", "EPGS:3857");
229 ignoreproj.add("EPSG:4267");
230 ignoreproj.add("EPSG:5221");
231 ignoreproj.add("EPSG:5514");
232 ignoreproj.add("EPSG:32019");
233 ignoreproj.add("EPSG:102066");
234 ignoreproj.add("EPSG:102067");
235 ignoreproj.add("EPSG:102685");
236 ignoreproj.add("EPSG:102711");
[13530]237 }
238
[15033]239 void loadSkip() throws IOException {
240 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
[15034]241 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
[15033]242 String line;
[11238]243
[15033]244 while ((line = fr.readLine()) != null) {
245 Matcher res = pattern.matcher(line);
246 if (res.matches()) {
247 if ("Ignore".equals(res.group(1))) {
248 skip.put(res.group(2), "green");
249 } else {
250 skip.put(res.group(2), "darkgoldenrod");
251 }
[11238]252 }
253 }
[11234]254 }
[11238]255 }
[9653]256
[15033]257 void myprintlnfinal(String s) throws IOException {
258 if (outputStream != null) {
259 outputStream.write(s + System.getProperty("line.separator"));
[9658]260 } else {
[15033]261 System.out.println(s);
[9658]262 }
263 }
264
[15033]265 void myprintln(String s) throws IOException {
266 if (skip.containsKey(s)) {
267 String color = skip.get(s);
268 skip.remove(s);
269 if (optionXhtmlBody || optionXhtml) {
[15034]270 s = "<pre style=\"margin:3px;color:"+color+"\">"
271 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9658]272 }
[15033]273 if (!optionNoSkip) {
274 return;
[9662]275 }
[15034]276 } else if (optionXhtmlBody || optionXhtml) {
[15033]277 String color =
278 s.startsWith("***") ? "black" :
279 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
280 (s.startsWith("#") ? "indigo" :
281 (s.startsWith("!") ? "orange" : "red")));
[15034]282 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9505]283 }
[15033]284 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
285 return;
[11965]286 }
[15033]287 myprintlnfinal(s);
[9658]288 }
289
[15033]290 void start() throws IOException {
291 if (optionXhtml) {
[15034]292 myprintlnfinal(
293 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
294 myprintlnfinal(
295 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
296 "<title>JOSM - ELI differences</title></head><body>\n");
[9505]297 }
298 }
[9653]299
[15033]300 void end() throws IOException {
301 for (String s : skip.keySet()) {
302 myprintln("+++ Obsolete skip entry: " + s);
[9658]303 }
[15033]304 if (optionXhtml) {
305 myprintlnfinal("</body></html>\n");
[9658]306 }
307 }
308
[15033]309 void loadELIEntries() throws IOException {
[15034]310 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
[15033]311 eliEntries = jr.readObject().getJsonArray("features");
312 }
[9653]313
[15033]314 for (JsonValue e : eliEntries) {
315 String url = getUrlStripped(e);
[9653]316 if (url.contains("{z}")) {
[15033]317 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
[15034]318 url = url.replace("{z}", "{zoom}");
[9653]319 }
[11582]320 if (eliUrls.containsKey(url)) {
[15033]321 myprintln("+++ ELI-URL is not unique: "+url);
[9505]322 } else {
[15033]323 eliUrls.put(url, e.asJsonObject());
[9505]324 }
[15033]325 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
326 if (s != null) {
327 String urlLc = url.toLowerCase(Locale.ENGLISH);
328 List<String> old = new LinkedList<>();
329 for (JsonValue p : s) {
330 String proj = ((JsonString) p).getString();
331 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
332 old.add(proj);
[13530]333 }
334 }
[15033]335 if (!old.isEmpty()) {
336 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
[13530]337 }
338 }
[7726]339 }
[15033]340 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
[7726]341 }
[15033]342
343 String cdata(String s) {
344 return cdata(s, false);
[11968]345 }
[7726]346
[15033]347 String cdata(String s, boolean escape) {
348 if (escape) {
349 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
350 } else if (s.matches("[<>&]"))
351 return "<![CDATA["+s+"]]>";
352 return s;
353 }
354
355 String maininfo(Object entry, String offset) {
356 String t = getType(entry);
357 String res = offset + "<type>"+t+"</type>\n";
358 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
359 if (getMinZoom(entry) != null)
360 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
361 if (getMaxZoom(entry) != null)
362 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
363 if ("wms".equals(t)) {
364 List<String> p = getProjections(entry);
365 if (p != null) {
366 res += offset + "<projections>\n";
[15034]367 for (String c : p) {
[15033]368 res += offset + " <code>"+c+"</code>\n";
[15034]369 }
[15033]370 res += offset + "</projections>\n";
[11975]371 }
[11967]372 }
[15033]373 return res;
[11967]374 }
[12061]375
[15033]376 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
377 DecimalFormat df = new DecimalFormat("#.#######");
378 df.setRoundingMode(java.math.RoundingMode.CEILING);
379 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
380 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
381 for (Object e : entries) {
[13536]382 stream.write(" <entry"
[15034]383 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
384 + (getOverlay(e) ? " overlay=\"true\"" : "")
[15033]385 + ">\n");
386 String t;
387 if (isNotBlank(t = getName(e)))
388 stream.write(" <name>"+cdata(t, true)+"</name>\n");
389 if (isNotBlank(t = getId(e)))
390 stream.write(" <id>"+t+"</id>\n");
391 if (isNotBlank(t = getCategory(e)))
392 stream.write(" <category>"+t+"</category>\n");
393 if (isNotBlank(t = getDate(e)))
394 stream.write(" <date>"+t+"</date>\n");
395 if (isNotBlank(t = getCountryCode(e)))
396 stream.write(" <country-code>"+t+"</country-code>\n");
397 if ((getDefault(e)))
398 stream.write(" <default>true</default>\n");
399 stream.write(maininfo(e, " "));
400 if (isNotBlank(t = getAttributionText(e)))
401 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
402 if (isNotBlank(t = getAttributionUrl(e)))
403 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
404 if (isNotBlank(t = getLogoImage(e)))
405 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
406 if (isNotBlank(t = getLogoUrl(e)))
407 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
408 if (isNotBlank(t = getTermsOfUseText(e)))
409 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
410 if (isNotBlank(t = getTermsOfUseUrl(e)))
411 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
412 if (isNotBlank(t = getPermissionReferenceUrl(e)))
413 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
414 if ((getValidGeoreference(e)))
415 stream.write(" <valid-georeference>true</valid-georeference>\n");
416 if (isNotBlank(t = getIcon(e)))
417 stream.write(" <icon>"+cdata(t)+"</icon>\n");
418 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
419 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
[11975]420 }
[15033]421 for (ImageryInfo m : getMirrors(e)) {
422 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
[11967]423 }
[15033]424 double minlat = 1000;
425 double minlon = 1000;
426 double maxlat = -1000;
427 double maxlon = -1000;
428 String shapes = "";
429 String sep = "\n ";
[13771]430 try {
[15033]431 for (Shape s: getShapes(e)) {
432 shapes += " <shape>";
433 int i = 0;
434 for (Coordinate p: s.getPoints()) {
435 double lat = p.getLat();
436 double lon = p.getLon();
437 if (lat > maxlat) maxlat = lat;
438 if (lon > maxlon) maxlon = lon;
439 if (lat < minlat) minlat = lat;
440 if (lon < minlon) minlon = lon;
[15034]441 if ((i++ % 3) == 0) {
[15033]442 shapes += sep + " ";
[13771]443 }
[15033]444 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
[11964]445 }
[15033]446 shapes += sep + "</shape>\n";
[11964]447 }
[15034]448 } catch (IllegalArgumentException ignored) {
449 Logging.trace(ignored);
[11964]450 }
[15033]451 if (!shapes.isEmpty()) {
[15034]452 stream.write(" <bounds min-lat='"+df.format(minlat)
453 +"' min-lon='"+df.format(minlon)
454 +"' max-lat='"+df.format(maxlat)
455 +"' max-lon='"+df.format(maxlon)+"'>\n");
[15033]456 stream.write(shapes + " </bounds>\n");
[11964]457 }
[15033]458 stream.write(" </entry>\n");
[11964]459 }
[15033]460 stream.write("</imagery>\n");
461 stream.close();
[11964]462 }
463
[15033]464 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
465 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
466 josmEntries = reader.parse();
467 }
[9667]468
[15033]469 for (ImageryInfo e : josmEntries) {
470 if (isBlank(getUrl(e))) {
471 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
472 continue;
[14554]473 }
[15033]474 if (isBlank(getName(e))) {
475 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
476 continue;
[14554]477 }
[15033]478 String url = getUrlStripped(e);
[9658]479 if (url.contains("{z}")) {
[15033]480 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
[15034]481 url = url.replace("{z}", "{zoom}");
[9658]482 }
[9505]483 if (josmUrls.containsKey(url)) {
[15033]484 myprintln("+++ JOSM-URL is not unique: "+url);
[9505]485 } else {
[15033]486 josmUrls.put(url, e);
[7726]487 }
[15033]488 for (ImageryInfo m : e.getMirrors()) {
489 url = getUrlStripped(m);
490 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
491 ReflectionUtils.setObjectsAccessible(origNameField);
[15034]492 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
[9658]493 if (josmUrls.containsKey(url)) {
[15033]494 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
[9658]495 } else {
[15033]496 josmUrls.put(url, m);
497 josmMirrors.put(url, m);
[9658]498 }
499 }
[7726]500 }
[15033]501 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
[7726]502 }
503
[15033]504 void checkInOneButNotTheOther() throws IOException {
505 List<String> le = new LinkedList<>(eliUrls.keySet());
506 List<String> lj = new LinkedList<>(josmUrls.keySet());
[13593]507
[15033]508 List<String> ke = new LinkedList<>(le);
509 for (String url : ke) {
510 if (lj.contains(url)) {
511 le.remove(url);
512 lj.remove(url);
[13593]513 }
514 }
515
[15033]516 if (!le.isEmpty() && !lj.isEmpty()) {
517 ke = new LinkedList<>(le);
518 for (String urle : ke) {
519 JsonObject e = eliUrls.get(urle);
520 String ide = getId(e);
[15034]521 String urlhttps = urle.replace("http:", "https:");
[15033]522 if (lj.contains(urlhttps)) {
523 myprintln("+ Missing https: "+getDescription(e));
524 eliUrls.put(urlhttps, eliUrls.get(urle));
525 eliUrls.remove(urle);
526 le.remove(urle);
527 lj.remove(urlhttps);
528 } else if (isNotBlank(ide)) {
529 List<String> kj = new LinkedList<>(lj);
530 for (String urlj : kj) {
531 ImageryInfo j = josmUrls.get(urlj);
532 String idj = getId(j);
[13714]533
[15033]534 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
535 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
536 le.remove(urle);
537 lj.remove(urlj);
538 // replace key for this entry with JOSM URL
539 eliUrls.remove(urle);
[15034]540 eliUrls.put(urlj, e);
[15033]541 break;
[13593]542 }
543 }
[13517]544 }
[7726]545 }
546 }
[13714]547
[15033]548 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
549 Collections.sort(le);
[13593]550 if (!le.isEmpty()) {
[15033]551 for (String l : le) {
552 myprintln("- " + getDescription(eliUrls.get(l)));
[11412]553 }
[7726]554 }
[15033]555 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
556 Collections.sort(lj);
[13593]557 if (!lj.isEmpty()) {
[15033]558 for (String l : lj) {
559 myprintln("+ " + getDescription(josmUrls.get(l)));
[11412]560 }
[7726]561 }
562 }
[9667]563
[15033]564 void checkCommonEntries() throws IOException {
[15034]565 doSameUrlButDifferentName();
566 doSameUrlButDifferentId();
567 doSameUrlButDifferentType();
568 doSameUrlButDifferentZoomBounds();
569 doSameUrlButDifferentCountryCode();
570 doSameUrlButDifferentQuality();
571 doSameUrlButDifferentDates();
572 doSameUrlButDifferentInformation();
573 doMismatchingShapes();
574 doMismatchingIcons();
575 doMiscellaneousChecks();
576 }
577
578 void doSameUrlButDifferentName() throws IOException {
[15033]579 myprintln("*** Same URL, but different name: ***");
580 for (String url : eliUrls.keySet()) {
581 JsonObject e = eliUrls.get(url);
582 if (!josmUrls.containsKey(url)) continue;
583 ImageryInfo j = josmUrls.get(url);
[15034]584 String ename = getName(e).replace("'", "\u2019");
585 String jname = getName(j).replace("'", "\u2019");
[11951]586 if (!ename.equals(jname)) {
[15033]587 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
[7726]588 }
589 }
[15034]590 }
[9667]591
[15034]592 void doSameUrlButDifferentId() throws IOException {
[15033]593 myprintln("*** Same URL, but different Id: ***");
594 for (String url : eliUrls.keySet()) {
595 JsonObject e = eliUrls.get(url);
596 if (!josmUrls.containsKey(url)) continue;
597 ImageryInfo j = josmUrls.get(url);
598 String ename = getId(e);
599 String jname = getId(j);
600 if (!Objects.equals(ename, jname)) {
601 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
[12126]602 }
[12226]603 }
[15034]604 }
[12126]605
[15034]606 void doSameUrlButDifferentType() throws IOException {
[15033]607 myprintln("*** Same URL, but different type: ***");
608 for (String url : eliUrls.keySet()) {
609 JsonObject e = eliUrls.get(url);
610 if (!josmUrls.containsKey(url)) continue;
611 ImageryInfo j = josmUrls.get(url);
612 if (!Objects.equals(getType(e), getType(j))) {
613 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
[7726]614 }
615 }
[15034]616 }
[9667]617
[15034]618 void doSameUrlButDifferentZoomBounds() throws IOException {
[15033]619 myprintln("*** Same URL, but different zoom bounds: ***");
620 for (String url : eliUrls.keySet()) {
621 JsonObject e = eliUrls.get(url);
622 if (!josmUrls.containsKey(url)) continue;
623 ImageryInfo j = josmUrls.get(url);
[7726]624
[15033]625 Integer eMinZoom = getMinZoom(e);
626 Integer jMinZoom = getMinZoom(j);
[13997]627 /* dont warn for entries copied from the base of the mirror */
[15033]628 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]629 jMinZoom = null;
[15033]630 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
631 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
[7726]632 }
[15033]633 Integer eMaxZoom = getMaxZoom(e);
634 Integer jMaxZoom = getMaxZoom(j);
[13997]635 /* dont warn for entries copied from the base of the mirror */
[15033]636 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]637 jMaxZoom = null;
[15033]638 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
639 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
[7726]640 }
641 }
[15034]642 }
[9667]643
[15034]644 void doSameUrlButDifferentCountryCode() throws IOException {
[15033]645 myprintln("*** Same URL, but different country code: ***");
646 for (String url : eliUrls.keySet()) {
647 JsonObject e = eliUrls.get(url);
648 if (!josmUrls.containsKey(url)) continue;
649 ImageryInfo j = josmUrls.get(url);
650 String cce = getCountryCode(e);
[13931]651 if ("ZZ".equals(cce)) { /* special ELI country code */
[15033]652 cce = null;
[13931]653 }
[15033]654 if (cce != null && !cce.equals(getCountryCode(j))) {
655 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
[7726]656 }
657 }
[15034]658 }
659
660 void doSameUrlButDifferentQuality() throws IOException {
[15033]661 myprintln("*** Same URL, but different quality: ***");
662 for (String url : eliUrls.keySet()) {
663 JsonObject e = eliUrls.get(url);
[9515]664 if (!josmUrls.containsKey(url)) {
[15033]665 String q = getQuality(e);
666 if ("eli-best".equals(q)) {
667 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
[9515]668 }
[15033]669 continue;
[9515]670 }
[15033]671 ImageryInfo j = josmUrls.get(url);
672 if (!Objects.equals(getQuality(e), getQuality(j))) {
673 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
[9505]674 }
[11599]675 }
[15034]676 }
677
678 void doSameUrlButDifferentDates() throws IOException {
[15033]679 myprintln("*** Same URL, but different dates: ***");
680 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
681 for (String url : eliUrls.keySet()) {
682 String ed = getDate(eliUrls.get(url));
683 if (!josmUrls.containsKey(url)) continue;
684 ImageryInfo j = josmUrls.get(url);
685 String jd = getDate(j);
[11612]686 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
[15034]687 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]688 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
[15033]689 String ed2 = ed;
690 Matcher m = pattern.matcher(ed);
691 if (m.matches()) {
692 Calendar cal = Calendar.getInstance();
693 cal.set(Integer.valueOf(m.group(2)),
694 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
695 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
696 cal.add(Calendar.DAY_OF_MONTH, -1);
697 ed2 = m.group(1) + cal.get(Calendar.YEAR);
698 if (m.group(4) != null)
699 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
700 if (m.group(6) != null)
701 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
[11639]702 }
[15034]703 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]704 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
[15033]705 String t = "'"+ed+"'";
[11612]706 if (!ed.equals(ef)) {
[15033]707 t += " or '"+ef+"'";
[11612]708 }
[11666]709 if (jd.isEmpty()) {
[15033]710 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
[11668]711 } else if (!ed.isEmpty()) {
[15033]712 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
713 } else if (!optionNoEli) {
714 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
[11666]715 }
[11573]716 }
[11665]717 }
[15034]718 }
719
720 void doSameUrlButDifferentInformation() throws IOException {
[15033]721 myprintln("*** Same URL, but different information: ***");
722 for (String url : eliUrls.keySet()) {
723 if (!josmUrls.containsKey(url)) continue;
724 JsonObject e = eliUrls.get(url);
725 ImageryInfo j = josmUrls.get(url);
[11981]726
[15082]727 compareDescriptions(e, j);
728 comparePermissionReferenceUrls(e, j);
729 compareAttributionUrls(e, j);
730 compareAttributionTexts(e, j);
731 compareProjections(e, j);
732 compareDefaults(e, j);
733 compareOverlays(e, j);
734 compareNoTileHeaders(e, j);
735 }
736 }
737
738 void compareDescriptions(JsonObject e, ImageryInfo j) throws IOException {
739 String et = getDescriptions(e).getOrDefault("en", "");
740 String jt = getDescriptions(j).getOrDefault("en", "");
741 if (!et.equals(jt)) {
742 if (jt.isEmpty()) {
743 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
744 } else if (!et.isEmpty()) {
745 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
746 } else if (!optionNoEli) {
747 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
[11981]748 }
[15082]749 }
750 }
[11981]751
[15082]752 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) throws IOException {
753 String et = getPermissionReferenceUrl(e);
754 String jt = getPermissionReferenceUrl(j);
755 String jt2 = getTermsOfUseUrl(j);
756 if (isBlank(jt)) jt = jt2;
757 if (!Objects.equals(et, jt)) {
758 if (isBlank(jt)) {
759 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
760 } else if (isNotBlank(et)) {
761 String ethttps = et.replace("http:", "https:");
762 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
763 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
764 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
765 } else {
766 String ja = getAttributionUrl(j);
767 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
768 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
[13578]769 } else {
[15082]770 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13578]771 }
[13551]772 }
[11981]773 }
[15082]774 } else if (!optionNoEli) {
775 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
[11981]776 }
[15082]777 }
778 }
[11981]779
[15082]780 void compareAttributionUrls(JsonObject e, ImageryInfo j) throws IOException {
781 String et = getAttributionUrl(e);
782 String jt = getAttributionUrl(j);
783 if (!Objects.equals(et, jt)) {
784 if (isBlank(jt)) {
785 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
786 } else if (isNotBlank(et)) {
787 String ethttps = et.replace("http:", "https:");
788 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
789 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
790 } else {
791 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[11981]792 }
[15082]793 } else if (!optionNoEli) {
794 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
[11981]795 }
[15082]796 }
797 }
[11981]798
[15082]799 void compareAttributionTexts(JsonObject e, ImageryInfo j) throws IOException {
800 String et = getAttributionText(e);
801 String jt = getAttributionText(j);
802 if (!Objects.equals(et, jt)) {
803 if (isBlank(jt)) {
804 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
805 } else if (isNotBlank(et)) {
806 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
807 } else if (!optionNoEli) {
808 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
[11981]809 }
[15082]810 }
811 }
[11981]812
[15082]813 void compareProjections(JsonObject e, ImageryInfo j) throws IOException {
814 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
815 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
816 if (!Objects.equals(et, jt)) {
817 if (isBlank(jt)) {
818 String t = getType(e);
819 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
820 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
821 } else {
822 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
[11981]823 }
[15082]824 } else if (isNotBlank(et)) {
825 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
826 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
827 } else {
828 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
829 }
830 } else if (!optionNoEli && !"tms".equals(getType(e))) {
831 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
[11981]832 }
[15082]833 }
834 }
[12226]835
[15082]836 void compareDefaults(JsonObject e, ImageryInfo j) throws IOException {
837 boolean ed = getDefault(e);
838 boolean jd = getDefault(j);
839 if (ed != jd) {
840 if (!jd) {
841 myprintln("- Missing JOSM default: "+getDescription(j));
842 } else if (!optionNoEli) {
843 myprintln("+ Missing ELI default: "+getDescription(j));
[12226]844 }
[15082]845 }
846 }
847
848 void compareOverlays(JsonObject e, ImageryInfo j) throws IOException {
849 boolean eo = getOverlay(e);
850 boolean jo = getOverlay(j);
851 if (eo != jo) {
852 if (!jo) {
853 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
854 } else if (!optionNoEli) {
855 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
[13536]856 }
[11981]857 }
[15034]858 }
859
[15082]860 void compareNoTileHeaders(JsonObject e, ImageryInfo j) throws IOException {
861 Map<String, Set<String>> eh = getNoTileHeader(e);
862 Map<String, Set<String>> jh = getNoTileHeader(j);
863 if (!Objects.equals(eh, jh)) {
864 if (jh == null || jh.isEmpty()) {
865 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
866 } else if (eh != null && !eh.isEmpty()) {
867 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
868 } else if (!optionNoEli) {
869 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
870 }
871 }
872 }
873
[15034]874 void doMismatchingShapes() throws IOException {
[15033]875 myprintln("*** Mismatching shapes: ***");
876 for (String url : josmUrls.keySet()) {
877 ImageryInfo j = josmUrls.get(url);
878 int num = 1;
879 for (Shape shape : getShapes(j)) {
880 List<Coordinate> p = shape.getPoints();
[15034]881 if (!p.get(0).equals(p.get(p.size()-1))) {
[15033]882 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
[11410]883 }
[15033]884 for (int nump = 1; nump < p.size(); ++nump) {
885 if (Objects.equals(p.get(nump-1), p.get(nump))) {
886 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
[11964]887 }
888 }
[15033]889 ++num;
[11410]890 }
891 }
[15033]892 for (String url : eliUrls.keySet()) {
893 JsonObject e = eliUrls.get(url);
894 int num = 1;
895 List<Shape> s = null;
[13771]896 try {
[15033]897 s = getShapes(e);
898 for (Shape shape : s) {
899 List<Coordinate> p = shape.getPoints();
[15034]900 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
[15033]901 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
[11964]902 }
[15033]903 for (int nump = 1; nump < p.size(); ++nump) {
904 if (Objects.equals(p.get(nump-1), p.get(nump))) {
905 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
[13771]906 }
907 }
[15033]908 ++num;
[11964]909 }
[15033]910 } catch (IllegalArgumentException err) {
911 String desc = getDescription(e);
912 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
[11410]913 }
[13780]914 if (s == null || !josmUrls.containsKey(url)) {
[15033]915 continue;
[11410]916 }
[15033]917 ImageryInfo j = josmUrls.get(url);
918 List<Shape> js = getShapes(j);
919 if (s.isEmpty() && !js.isEmpty()) {
920 if (!optionNoEli) {
921 myprintln("+ No ELI shape: "+getDescription(j));
[11414]922 }
[15034]923 } else if (js.isEmpty() && !s.isEmpty()) {
[11415]924 // don't report boundary like 5 point shapes as difference
[15033]925 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
926 myprintln("- No JOSM shape: "+getDescription(j));
[11415]927 }
[15034]928 } else if (s.size() != js.size()) {
[15033]929 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
[11413]930 } else {
[15033]931 boolean[] edone = new boolean[s.size()];
932 boolean[] jdone = new boolean[js.size()];
933 for (int enums = 0; enums < s.size(); ++enums) {
934 List<Coordinate> ep = s.get(enums).getPoints();
935 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
936 List<Coordinate> jp = js.get(jnums).getPoints();
937 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]938 boolean err = false;
[15034]939 for (int nump = 0; nump < ep.size() && !err; ++nump) {
[15033]940 Coordinate ept = ep.get(nump);
941 Coordinate jpt = jp.get(nump);
[15034]942 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
[15033]943 err = true;
[11413]944 }
[15034]945 if (!err) {
[15033]946 edone[enums] = true;
947 jdone[jnums] = true;
948 break;
[14576]949 }
[11413]950 }
951 }
[11411]952 }
[15033]953 for (int enums = 0; enums < s.size(); ++enums) {
954 List<Coordinate> ep = s.get(enums).getPoints();
955 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
956 List<Coordinate> jp = js.get(jnums).getPoints();
957 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]958 boolean err = false;
[15033]959 for (int nump = 0; nump < ep.size() && !err; ++nump) {
960 Coordinate ept = ep.get(nump);
961 Coordinate jpt = jp.get(nump);
962 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
963 String numtxt = Integer.toString(enums+1);
964 if (enums != jnums) {
965 numtxt += '/' + Integer.toString(jnums+1);
966 }
967 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
968 break;
[14576]969 }
970 }
[15033]971 edone[enums] = true;
972 jdone[jnums] = true;
973 break;
[14576]974 }
975 }
976 }
[15033]977 for (int enums = 0; enums < s.size(); ++enums) {
978 List<Coordinate> ep = s.get(enums).getPoints();
979 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
980 List<Coordinate> jp = js.get(jnums).getPoints();
981 if (!jdone[jnums]) {
982 String numtxt = Integer.toString(enums+1);
983 if (enums != jnums) {
984 numtxt += '/' + Integer.toString(jnums+1);
985 }
[15034]986 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
987 + getDescription(j));
[15033]988 edone[enums] = true;
989 jdone[jnums] = true;
990 break;
[14576]991 }
992 }
993 }
[11410]994 }
995 }
[15034]996 }
997
998 void doMismatchingIcons() throws IOException {
[15033]999 myprintln("*** Mismatching icons: ***");
1000 for (String url : eliUrls.keySet()) {
1001 JsonObject e = eliUrls.get(url);
[11420]1002 if (!josmUrls.containsKey(url)) {
[15033]1003 continue;
[11420]1004 }
[15033]1005 ImageryInfo j = josmUrls.get(url);
1006 String ij = getIcon(j);
1007 String ie = getIcon(e);
1008 boolean ijok = isNotBlank(ij);
1009 boolean ieok = isNotBlank(ie);
1010 if (ijok && !ieok) {
1011 if (!optionNoEli) {
1012 myprintln("+ No ELI icon: "+getDescription(j));
[11420]1013 }
[15033]1014 } else if (!ijok && ieok) {
1015 myprintln("- No JOSM icon: "+getDescription(j));
1016 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
[14351]1017 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1018 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
[14293]1019 ij.startsWith("data:"))) {
[15034]1020 String iehttps = ie.replace("http:", "https:");
[15033]1021 if (ij.equals(iehttps)) {
1022 myprintln("+ Different icons: "+getDescription(j));
[14294]1023 } else {
[15033]1024 myprintln("* Different icons: "+getDescription(j));
[14294]1025 }
[11420]1026 }
1027 }
[15034]1028 }
1029
1030 void doMiscellaneousChecks() throws IOException {
[15033]1031 myprintln("*** Miscellaneous checks: ***");
1032 Map<String, ImageryInfo> josmIds = new HashMap<>();
1033 Collection<String> all = Projections.getAllProjectionCodes();
[13551]1034 DomainValidator dv = DomainValidator.getInstance();
[15033]1035 for (String url : josmUrls.keySet()) {
1036 ImageryInfo j = josmUrls.get(url);
1037 String id = getId(j);
1038 if ("wms".equals(getType(j))) {
1039 String urlLc = url.toLowerCase(Locale.ENGLISH);
1040 if (getProjections(j).isEmpty()) {
1041 myprintln("* WMS without projections: "+getDescription(j));
[13526]1042 } else {
[15033]1043 List<String> unsupported = new LinkedList<>();
1044 List<String> old = new LinkedList<>();
1045 for (String p : getProjectionsUnstripped(j)) {
1046 if ("CRS:84".equals(p)) {
1047 if (!urlLc.contains("version=1.3")) {
1048 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
[13526]1049 }
[15033]1050 } else if (oldproj.containsKey(p)) {
1051 old.add(p);
1052 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1053 unsupported.add(p);
[13526]1054 }
1055 }
[15033]1056 if (!unsupported.isEmpty()) {
1057 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
[13526]1058 }
[15033]1059 for (String o : old) {
[15034]1060 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1061 + getDescription(j));
[13527]1062 }
[13526]1063 }
[15033]1064 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1065 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1066 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1067 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
[13526]1068 }
[13511]1069 }
[15033]1070 List<String> urls = new LinkedList<>();
1071 if (!"scanex".equals(getType(j))) {
1072 urls.add(url);
[13532]1073 }
[15033]1074 String jt = getPermissionReferenceUrl(j);
1075 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1076 urls.add(jt);
1077 jt = getTermsOfUseUrl(j);
1078 if (isNotBlank(jt))
1079 urls.add(jt);
1080 jt = getAttributionUrl(j);
1081 if (isNotBlank(jt))
1082 urls.add(jt);
1083 jt = getIcon(j);
[15439]1084 if (isNotBlank(jt)) {
1085 if(!jt.startsWith("data:image/"))
1086 urls.add(jt);
1087 else {
1088 try
1089 {
1090 new ImageProvider(jt).get();
1091 } catch(Exception e) {
1092 myprintln("* Strange Icon: "+getDescription(j));
1093 }
1094 }
1095 }
[15163]1096 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
[15033]1097 for (String u : urls) {
[15163]1098 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
[15033]1099 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1100 } else {
[15163]1101 try {
1102 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1103 String domain = jurl.getHost();
1104 int port = jurl.getPort();
1105 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1106 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1107 else if (80 == port || 443 == port) {
1108 myprintln("* Useless port '"+port+"': "+getDescription(j));
1109 }
1110 } catch (MalformedURLException e) {
1111 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
[13554]1112 }
[13551]1113 }
1114 }
1115
[15033]1116 if (josmMirrors.containsKey(url)) {
1117 continue;
[11420]1118 }
[15033]1119 if (isBlank(id)) {
1120 myprintln("* No JOSM-ID: "+getDescription(j));
1121 } else if (josmIds.containsKey(id)) {
1122 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
[11420]1123 } else {
[15033]1124 josmIds.put(id, j);
[11420]1125 }
[15033]1126 String d = getDate(j);
1127 if (isNotBlank(d)) {
1128 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1129 Matcher m = patternD.matcher(d);
1130 if (!m.matches()) {
1131 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
[11572]1132 } else {
1133 try {
[15033]1134 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1135 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1136 if (second.compareTo(first) < 0) {
1137 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
[11572]1138 }
[15034]1139 } catch (Exception e) {
[15033]1140 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
[11572]1141 }
1142 }
[11603]1143 }
[15033]1144 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1145 myprintln("* Attribution link without text: "+getDescription(j));
[12261]1146 }
[15033]1147 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1148 myprintln("* Logo link without image: "+getDescription(j));
[12261]1149 }
[15033]1150 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1151 myprintln("* Terms of Use text without link: "+getDescription(j));
[12261]1152 }
[15033]1153 List<Shape> js = getShapes(j);
1154 if (!js.isEmpty()) {
1155 double minlat = 1000;
1156 double minlon = 1000;
1157 double maxlat = -1000;
1158 double maxlon = -1000;
1159 for (Shape s: js) {
1160 for (Coordinate p: s.getPoints()) {
1161 double lat = p.getLat();
1162 double lon = p.getLon();
[15034]1163 if (lat > maxlat) maxlat = lat;
1164 if (lon > maxlon) maxlon = lon;
1165 if (lat < minlat) minlat = lat;
1166 if (lon < minlon) minlon = lon;
[11422]1167 }
1168 }
[15033]1169 ImageryBounds b = j.getBounds();
1170 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1171 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
[15034]1172 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1173 + getDescription(j));
[11422]1174 }
1175 }
[15033]1176 List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
1177 String cat = getCategory(j);
1178 if (isBlank(cat)) {
1179 myprintln("* No category: "+getDescription(j));
1180 } else if (!knownCategories.contains(cat)) {
1181 myprintln("* Strange category "+cat+": "+getDescription(j));
[13792]1182 }
[11420]1183 }
[7726]1184 }
[9667]1185
[15033]1186 /*
[7726]1187 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1188 */
[15033]1189
[7726]1190 static String getUrl(Object e) {
[15033]1191 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1192 return ((Map<String, JsonObject>) e).get("properties").getString("url");
[7726]1193 }
[15033]1194
[12242]1195 static String getUrlStripped(Object e) {
[15034]1196 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
[12242]1197 }
[15033]1198
[11572]1199 static String getDate(Object e) {
[15033]1200 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1201 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1202 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1203 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
[15034]1204 if (!start.isEmpty() && !end.isEmpty())
[15033]1205 return start+";"+end;
[15034]1206 else if (!start.isEmpty())
[15033]1207 return start+";-";
[15034]1208 else if (!end.isEmpty())
[15033]1209 return "-;"+end;
1210 return "";
[11572]1211 }
[15033]1212
1213 static Date verifyDate(String year, String month, String day) throws ParseException {
1214 String date;
[15034]1215 if (year == null) {
[15033]1216 date = "3000-01-01";
[11854]1217 } else {
[15033]1218 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
[11854]1219 }
[15033]1220 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1221 df.setLenient(false);
1222 return df.parse(date);
[11572]1223 }
[15033]1224
[11420]1225 static String getId(Object e) {
[15033]1226 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1227 return ((Map<String, JsonObject>) e).get("properties").getString("id");
[11420]1228 }
[15033]1229
[7726]1230 static String getName(Object e) {
[15033]1231 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1232 return ((Map<String, JsonObject>) e).get("properties").getString("name");
[7726]1233 }
[15033]1234
1235 static List<ImageryInfo> getMirrors(Object e) {
1236 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1237 return Collections.emptyList();
[11967]1238 }
[15033]1239
1240 static List<String> getProjections(Object e) {
1241 List<String> r = new ArrayList<>();
1242 List<String> u = getProjectionsUnstripped(e);
1243 if (u != null) {
1244 for (String p : u) {
1245 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1246 r.add(p);
[13530]1247 }
1248 }
1249 }
[15033]1250 return r;
[13530]1251 }
[15033]1252
1253 static List<String> getProjectionsUnstripped(Object e) {
1254 List<String> r = null;
[11975]1255 if (e instanceof ImageryInfo) {
[15033]1256 r = ((ImageryInfo) e).getServerProjections();
[11975]1257 } else {
[15033]1258 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1259 if (s != null) {
1260 r = new ArrayList<>();
1261 for (JsonValue p : s.asJsonArray()) {
1262 r.add(((JsonString) p).getString());
[13530]1263 }
[11981]1264 }
[11975]1265 }
[15033]1266 return r != null ? r : Collections.emptyList();
[11975]1267 }
[15033]1268
[11410]1269 static List<Shape> getShapes(Object e) {
1270 if (e instanceof ImageryInfo) {
[15033]1271 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
[15034]1272 if (bounds != null) {
[15033]1273 return bounds.getShapes();
[11411]1274 }
[15033]1275 return Collections.emptyList();
[11410]1276 }
[15033]1277 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1278 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1279 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1280 List<Shape> l = new ArrayList<>();
1281 for (JsonValue shapes: poly) {
1282 Shape s = new Shape();
1283 for (JsonValue point: shapes.asJsonArray()) {
1284 String lon = point.asJsonArray().getJsonNumber(0).toString();
1285 String lat = point.asJsonArray().getJsonNumber(1).toString();
1286 s.addPoint(lat, lon);
[11410]1287 }
[15033]1288 l.add(s);
[11410]1289 }
[15033]1290 return l;
[11410]1291 }
[15033]1292 return Collections.emptyList();
[11410]1293 }
[15033]1294
[7726]1295 static String getType(Object e) {
[15033]1296 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1297 return ((Map<String, JsonObject>) e).get("properties").getString("type");
[7726]1298 }
[15033]1299
[7726]1300 static Integer getMinZoom(Object e) {
1301 if (e instanceof ImageryInfo) {
[15033]1302 int mz = ((ImageryInfo) e).getMinZoom();
1303 return mz == 0 ? null : mz;
[7726]1304 } else {
[15033]1305 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1306 if (num == null) return null;
1307 return num.intValue();
[7726]1308 }
1309 }
[15033]1310
[7726]1311 static Integer getMaxZoom(Object e) {
1312 if (e instanceof ImageryInfo) {
[15033]1313 int mz = ((ImageryInfo) e).getMaxZoom();
1314 return mz == 0 ? null : mz;
[7726]1315 } else {
[15033]1316 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1317 if (num == null) return null;
1318 return num.intValue();
[7726]1319 }
1320 }
[15033]1321
[7726]1322 static String getCountryCode(Object e) {
[15033]1323 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1324 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
[7726]1325 }
[15033]1326
[9505]1327 static String getQuality(Object e) {
[15033]1328 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1329 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1330 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
[9505]1331 }
[15033]1332
1333 static boolean getOverlay(Object e) {
1334 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1335 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1336 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
[13536]1337 }
[15033]1338
[11420]1339 static String getIcon(Object e) {
[15033]1340 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1341 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
[11420]1342 }
[15033]1343
[11975]1344 static String getAttributionText(Object e) {
[15033]1345 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1346 try {
1347 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1348 } catch (NullPointerException ex) {
1349 return null;
1350 }
[11975]1351 }
[15033]1352
[11975]1353 static String getAttributionUrl(Object e) {
[15033]1354 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1355 try {
1356 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1357 } catch (NullPointerException ex) {
1358 return null;
1359 }
[11975]1360 }
[15033]1361
[11975]1362 static String getTermsOfUseText(Object e) {
[15033]1363 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1364 return null;
[11975]1365 }
[15033]1366
[11975]1367 static String getTermsOfUseUrl(Object e) {
[15033]1368 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1369 return null;
[11975]1370 }
[15033]1371
[13792]1372 static String getCategory(Object e) {
1373 if (e instanceof ImageryInfo) {
[15033]1374 return ((ImageryInfo) e).getImageryCategoryOriginalString();
[13792]1375 }
[15033]1376 return null;
[13792]1377 }
[15033]1378
[12261]1379 static String getLogoImage(Object e) {
[15033]1380 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1381 return null;
[12261]1382 }
[15033]1383
[12261]1384 static String getLogoUrl(Object e) {
[15033]1385 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1386 return null;
[12261]1387 }
[15033]1388
[11975]1389 static String getPermissionReferenceUrl(Object e) {
[15033]1390 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1391 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
[11975]1392 }
[15033]1393
[15082]1394 static Map<String, Set<String>> getNoTileHeader(Object e) {
1395 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1396 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1397 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1398 Function.identity(),
1399 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1400 }
1401
[15034]1402 static Map<String, String> getDescriptions(Object e) {
[15082]1403 Map<String, String> res = new HashMap<>();
[11975]1404 if (e instanceof ImageryInfo) {
[15033]1405 String a = ((ImageryInfo) e).getDescription();
1406 if (a != null) res.put("en", a);
[11975]1407 } else {
[15033]1408 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
[15034]1409 if (a != null) res.put("en", a.replaceAll("''", "'"));
[11975]1410 }
[15033]1411 return res;
[11975]1412 }
[15033]1413
1414 static boolean getValidGeoreference(Object e) {
1415 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1416 return false;
[11975]1417 }
[15033]1418
1419 static boolean getDefault(Object e) {
1420 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1421 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
[12226]1422 }
[15033]1423
[7726]1424 String getDescription(Object o) {
[15033]1425 String url = getUrl(o);
1426 String cc = getCountryCode(o);
[7726]1427 if (cc == null) {
[15033]1428 ImageryInfo j = josmUrls.get(url);
1429 if (j != null) cc = getCountryCode(j);
[7726]1430 if (cc == null) {
[15033]1431 JsonObject e = eliUrls.get(url);
1432 if (e != null) cc = getCountryCode(e);
[7726]1433 }
1434 }
1435 if (cc == null) {
[15033]1436 cc = "";
[7726]1437 } else {
[15033]1438 cc = "["+cc+"] ";
[7726]1439 }
[15033]1440 String d = cc + getName(o) + " - " + getUrl(o);
1441 if (optionShorten) {
1442 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
[7726]1443 }
[15033]1444 return d;
[7726]1445 }
1446}
Note: See TracBrowser for help on using the repository browser.