source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15134

Last change on this file since 15134 was 15082, checked in by Don-vip, 7 years ago

see #17730 - compare no tile headers, refactor
  • Property svn:eol-style set to native
File size: 62.6 KB
RevLine 
[7726]1// License: GPL. For details, see LICENSE file.
[15034]2import static java.nio.charset.StandardCharsets.UTF_8;
[15033]3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
[15034]9import java.io.OutputStream;
[15033]10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
[15034]12import java.nio.file.Files;
13import java.nio.file.Paths;
[15033]14import java.text.DecimalFormat;
15import java.text.ParseException;
16import java.text.SimpleDateFormat;
17import java.util.ArrayList;
18import java.util.Arrays;
19import java.util.Calendar;
20import java.util.Collection;
21import java.util.Collections;
22import java.util.Date;
23import java.util.HashMap;
24import java.util.LinkedList;
25import java.util.List;
26import java.util.Locale;
27import java.util.Map;
28import java.util.Map.Entry;
29import java.util.Objects;
[15082]30import java.util.Set;
31import java.util.function.Function;
[15033]32import java.util.regex.Matcher;
33import java.util.regex.Pattern;
34import java.util.stream.Collectors;
35
36import javax.json.Json;
37import javax.json.JsonArray;
38import javax.json.JsonNumber;
39import javax.json.JsonObject;
40import javax.json.JsonReader;
41import javax.json.JsonString;
42import javax.json.JsonValue;
43
44import org.openstreetmap.gui.jmapviewer.Coordinate;
45import org.openstreetmap.josm.data.Preferences;
46import org.openstreetmap.josm.data.imagery.ImageryInfo;
47import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
48import org.openstreetmap.josm.data.imagery.Shape;
49import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
50import org.openstreetmap.josm.data.projection.Projections;
51import org.openstreetmap.josm.data.validation.routines.DomainValidator;
52import org.openstreetmap.josm.io.imagery.ImageryReader;
53import org.openstreetmap.josm.spi.preferences.Config;
[15034]54import org.openstreetmap.josm.tools.Logging;
[15033]55import org.openstreetmap.josm.tools.OptionParser;
56import org.openstreetmap.josm.tools.OptionParser.OptionCount;
57import org.openstreetmap.josm.tools.ReflectionUtils;
58import org.xml.sax.SAXException;
59
[7726]60/**
[11854]61 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
[7726]62 * The goal is to keep both lists in sync.
63 *
[11854]64 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
[11694]65 * provides also a version in the JOSM format, but the GEOJSON is the original source
[7726]66 * format, so we read that.
67 *
68 * How to run:
69 * -----------
70 *
71 * Main JOSM binary needs to be in classpath, e.g.
72 *
[15033]73 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
[9667]74 *
[7726]75 * Add option "-h" to show the available command line flags.
76 */
[15037]77@SuppressWarnings("unchecked")
[15033]78public class SyncEditorLayerIndex {
[14019]79
[15034]80 private static final int MAXLEN = 140;
81
[15033]82 private List<ImageryInfo> josmEntries;
83 private JsonArray eliEntries;
[7726]84
[15034]85 private final Map<String, JsonObject> eliUrls = new HashMap<>();
86 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
87 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
88 private static final Map<String, String> oldproj = new HashMap<>();
89 private static final List<String> ignoreproj = new LinkedList<>();
[7726]90
[15033]91 private static String eliInputFile = "imagery_eli.geojson";
92 private static String josmInputFile = "imagery_josm.imagery.xml";
93 private static String ignoreInputFile = "imagery_josm.ignores.txt";
[15034]94 private static OutputStream outputFile;
95 private static OutputStreamWriter outputStream;
[15033]96 private static String optionOutput;
97 private static boolean optionShorten;
98 private static boolean optionNoSkip;
99 private static boolean optionXhtmlBody;
100 private static boolean optionXhtml;
101 private static String optionEliXml;
102 private static String optionJosmXml;
103 private static String optionEncoding;
104 private static boolean optionNoEli;
105 private Map<String, String> skip = new HashMap<>();
[7726]106
107 /**
108 * Main method.
[15033]109 * @param args program arguments
110 * @throws IOException if any I/O error occurs
111 * @throws ReflectiveOperationException if any reflective operation error occurs
112 * @throws SAXException if any SAX error occurs
[7726]113 */
[15033]114 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
115 Locale.setDefault(Locale.ROOT);
[15034]116 parseCommandLineArguments(args);
[15033]117 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
118 Config.setPreferencesInstance(pref);
119 pref.init(false);
120 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
121 script.setupProj();
122 script.loadSkip();
123 script.start();
124 script.loadJosmEntries();
125 if (optionJosmXml != null) {
[15034]126 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
127 script.printentries(script.josmEntries, stream);
128 }
[11964]129 }
[15033]130 script.loadELIEntries();
131 if (optionEliXml != null) {
[15034]132 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
133 script.printentries(script.eliEntries, stream);
134 }
[11964]135 }
[15033]136 script.checkInOneButNotTheOther();
137 script.checkCommonEntries();
138 script.end();
139 if (outputStream != null) {
140 outputStream.close();
[9505]141 }
[15033]142 if (outputFile != null) {
143 outputFile.close();
[9505]144 }
[7726]145 }
[9653]146
[7726]147 /**
[15033]148 * Displays help on the console
149 */
150 private static void showHelp() {
151 System.out.println(getHelp());
152 System.exit(0);
153 }
154
155 static String getHelp() {
156 return "usage: java -cp build SyncEditorLayerIndex\n" +
157 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
[15034]158 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
159 "Default is imagery_eli.geojson (current directory).\n" +
[15033]160 "-h,--help show this help\n" +
161 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
[15034]162 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
163 "Default is imagery_josm.imagery.xml (current directory).\n" +
[15033]164 "-m,--noeli don't show output for ELI problems\n" +
165 "-n,--noskip don't skip known entries\n" +
166 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
167 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
168 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
169 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
170 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
171 "-X,--xhtml create XHTML for display in a web page\n";
172 }
173
174 /**
[7726]175 * Parse command line arguments.
[15033]176 * @param args program arguments
[15034]177 * @throws IOException in case of I/O error
[7726]178 */
[15034]179 static void parseCommandLineArguments(String[] args) throws IOException {
[15033]180 new OptionParser("JOSM/ELI synchronization script")
181 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
182 .addShortAlias("help", "h")
183 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
184 .addShortAlias("output", "o")
185 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
186 .addShortAlias("eli_input", "e")
187 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
188 .addShortAlias("josm_input", "j")
189 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
190 .addShortAlias("ignore_input", "i")
191 .addFlagParameter("shorten", () -> optionShorten = true)
192 .addShortAlias("shorten", "s")
193 .addFlagParameter("noskip", () -> optionNoSkip = true)
194 .addShortAlias("noskip", "n")
195 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
196 .addShortAlias("xhtmlbody", "x")
197 .addFlagParameter("xhtml", () -> optionXhtml = true)
198 .addShortAlias("xhtml", "X")
199 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
200 .addShortAlias("elixml", "p")
201 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
202 .addShortAlias("josmxml", "q")
203 .addFlagParameter("noeli", () -> optionNoEli = true)
204 .addShortAlias("noeli", "m")
205 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
206 .addShortAlias("encoding", "c")
207 .parseOptionsOrExit(Arrays.asList(args));
[7726]208
[15034]209 if (optionOutput != null && !"-".equals(optionOutput)) {
210 outputFile = Files.newOutputStream(Paths.get(optionOutput));
[15033]211 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
212 } else if (optionEncoding != null) {
213 outputStream = new OutputStreamWriter(System.out, optionEncoding);
[7726]214 }
215 }
216
[13530]217 void setupProj() {
[15033]218 oldproj.put("EPSG:3359", "EPSG:3404");
219 oldproj.put("EPSG:3785", "EPSG:3857");
220 oldproj.put("EPSG:31297", "EPGS:31287");
221 oldproj.put("EPSG:31464", "EPSG:31468");
222 oldproj.put("EPSG:54004", "EPSG:3857");
223 oldproj.put("EPSG:102100", "EPSG:3857");
224 oldproj.put("EPSG:102113", "EPSG:3857");
225 oldproj.put("EPSG:900913", "EPGS:3857");
226 ignoreproj.add("EPSG:4267");
227 ignoreproj.add("EPSG:5221");
228 ignoreproj.add("EPSG:5514");
229 ignoreproj.add("EPSG:32019");
230 ignoreproj.add("EPSG:102066");
231 ignoreproj.add("EPSG:102067");
232 ignoreproj.add("EPSG:102685");
233 ignoreproj.add("EPSG:102711");
[13530]234 }
235
[15033]236 void loadSkip() throws IOException {
237 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
[15034]238 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
[15033]239 String line;
[11238]240
[15033]241 while ((line = fr.readLine()) != null) {
242 Matcher res = pattern.matcher(line);
243 if (res.matches()) {
244 if ("Ignore".equals(res.group(1))) {
245 skip.put(res.group(2), "green");
246 } else {
247 skip.put(res.group(2), "darkgoldenrod");
248 }
[11238]249 }
250 }
[11234]251 }
[11238]252 }
[9653]253
[15033]254 void myprintlnfinal(String s) throws IOException {
255 if (outputStream != null) {
256 outputStream.write(s + System.getProperty("line.separator"));
[9658]257 } else {
[15033]258 System.out.println(s);
[9658]259 }
260 }
261
[15033]262 void myprintln(String s) throws IOException {
263 if (skip.containsKey(s)) {
264 String color = skip.get(s);
265 skip.remove(s);
266 if (optionXhtmlBody || optionXhtml) {
[15034]267 s = "<pre style=\"margin:3px;color:"+color+"\">"
268 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9658]269 }
[15033]270 if (!optionNoSkip) {
271 return;
[9662]272 }
[15034]273 } else if (optionXhtmlBody || optionXhtml) {
[15033]274 String color =
275 s.startsWith("***") ? "black" :
276 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
277 (s.startsWith("#") ? "indigo" :
278 (s.startsWith("!") ? "orange" : "red")));
[15034]279 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9505]280 }
[15033]281 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
282 return;
[11965]283 }
[15033]284 myprintlnfinal(s);
[9658]285 }
286
[15033]287 void start() throws IOException {
288 if (optionXhtml) {
[15034]289 myprintlnfinal(
290 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
291 myprintlnfinal(
292 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
293 "<title>JOSM - ELI differences</title></head><body>\n");
[9505]294 }
295 }
[9653]296
[15033]297 void end() throws IOException {
298 for (String s : skip.keySet()) {
299 myprintln("+++ Obsolete skip entry: " + s);
[9658]300 }
[15033]301 if (optionXhtml) {
302 myprintlnfinal("</body></html>\n");
[9658]303 }
304 }
305
[15033]306 void loadELIEntries() throws IOException {
[15034]307 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
[15033]308 eliEntries = jr.readObject().getJsonArray("features");
309 }
[9653]310
[15033]311 for (JsonValue e : eliEntries) {
312 String url = getUrlStripped(e);
[9653]313 if (url.contains("{z}")) {
[15033]314 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
[15034]315 url = url.replace("{z}", "{zoom}");
[9653]316 }
[11582]317 if (eliUrls.containsKey(url)) {
[15033]318 myprintln("+++ ELI-URL is not unique: "+url);
[9505]319 } else {
[15033]320 eliUrls.put(url, e.asJsonObject());
[9505]321 }
[15033]322 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
323 if (s != null) {
324 String urlLc = url.toLowerCase(Locale.ENGLISH);
325 List<String> old = new LinkedList<>();
326 for (JsonValue p : s) {
327 String proj = ((JsonString) p).getString();
328 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
329 old.add(proj);
[13530]330 }
331 }
[15033]332 if (!old.isEmpty()) {
333 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
[13530]334 }
335 }
[7726]336 }
[15033]337 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
[7726]338 }
[15033]339
340 String cdata(String s) {
341 return cdata(s, false);
[11968]342 }
[7726]343
[15033]344 String cdata(String s, boolean escape) {
345 if (escape) {
346 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
347 } else if (s.matches("[<>&]"))
348 return "<![CDATA["+s+"]]>";
349 return s;
350 }
351
352 String maininfo(Object entry, String offset) {
353 String t = getType(entry);
354 String res = offset + "<type>"+t+"</type>\n";
355 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
356 if (getMinZoom(entry) != null)
357 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
358 if (getMaxZoom(entry) != null)
359 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
360 if ("wms".equals(t)) {
361 List<String> p = getProjections(entry);
362 if (p != null) {
363 res += offset + "<projections>\n";
[15034]364 for (String c : p) {
[15033]365 res += offset + " <code>"+c+"</code>\n";
[15034]366 }
[15033]367 res += offset + "</projections>\n";
[11975]368 }
[11967]369 }
[15033]370 return res;
[11967]371 }
[12061]372
[15033]373 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
374 DecimalFormat df = new DecimalFormat("#.#######");
375 df.setRoundingMode(java.math.RoundingMode.CEILING);
376 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
377 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
378 for (Object e : entries) {
[13536]379 stream.write(" <entry"
[15034]380 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
381 + (getOverlay(e) ? " overlay=\"true\"" : "")
[15033]382 + ">\n");
383 String t;
384 if (isNotBlank(t = getName(e)))
385 stream.write(" <name>"+cdata(t, true)+"</name>\n");
386 if (isNotBlank(t = getId(e)))
387 stream.write(" <id>"+t+"</id>\n");
388 if (isNotBlank(t = getCategory(e)))
389 stream.write(" <category>"+t+"</category>\n");
390 if (isNotBlank(t = getDate(e)))
391 stream.write(" <date>"+t+"</date>\n");
392 if (isNotBlank(t = getCountryCode(e)))
393 stream.write(" <country-code>"+t+"</country-code>\n");
394 if ((getDefault(e)))
395 stream.write(" <default>true</default>\n");
396 stream.write(maininfo(e, " "));
397 if (isNotBlank(t = getAttributionText(e)))
398 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
399 if (isNotBlank(t = getAttributionUrl(e)))
400 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
401 if (isNotBlank(t = getLogoImage(e)))
402 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
403 if (isNotBlank(t = getLogoUrl(e)))
404 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
405 if (isNotBlank(t = getTermsOfUseText(e)))
406 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
407 if (isNotBlank(t = getTermsOfUseUrl(e)))
408 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
409 if (isNotBlank(t = getPermissionReferenceUrl(e)))
410 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
411 if ((getValidGeoreference(e)))
412 stream.write(" <valid-georeference>true</valid-georeference>\n");
413 if (isNotBlank(t = getIcon(e)))
414 stream.write(" <icon>"+cdata(t)+"</icon>\n");
415 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
416 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
[11975]417 }
[15033]418 for (ImageryInfo m : getMirrors(e)) {
419 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
[11967]420 }
[15033]421 double minlat = 1000;
422 double minlon = 1000;
423 double maxlat = -1000;
424 double maxlon = -1000;
425 String shapes = "";
426 String sep = "\n ";
[13771]427 try {
[15033]428 for (Shape s: getShapes(e)) {
429 shapes += " <shape>";
430 int i = 0;
431 for (Coordinate p: s.getPoints()) {
432 double lat = p.getLat();
433 double lon = p.getLon();
434 if (lat > maxlat) maxlat = lat;
435 if (lon > maxlon) maxlon = lon;
436 if (lat < minlat) minlat = lat;
437 if (lon < minlon) minlon = lon;
[15034]438 if ((i++ % 3) == 0) {
[15033]439 shapes += sep + " ";
[13771]440 }
[15033]441 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
[11964]442 }
[15033]443 shapes += sep + "</shape>\n";
[11964]444 }
[15034]445 } catch (IllegalArgumentException ignored) {
446 Logging.trace(ignored);
[11964]447 }
[15033]448 if (!shapes.isEmpty()) {
[15034]449 stream.write(" <bounds min-lat='"+df.format(minlat)
450 +"' min-lon='"+df.format(minlon)
451 +"' max-lat='"+df.format(maxlat)
452 +"' max-lon='"+df.format(maxlon)+"'>\n");
[15033]453 stream.write(shapes + " </bounds>\n");
[11964]454 }
[15033]455 stream.write(" </entry>\n");
[11964]456 }
[15033]457 stream.write("</imagery>\n");
458 stream.close();
[11964]459 }
460
[15033]461 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
462 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
463 josmEntries = reader.parse();
464 }
[9667]465
[15033]466 for (ImageryInfo e : josmEntries) {
467 if (isBlank(getUrl(e))) {
468 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
469 continue;
[14554]470 }
[15033]471 if (isBlank(getName(e))) {
472 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
473 continue;
[14554]474 }
[15033]475 String url = getUrlStripped(e);
[9658]476 if (url.contains("{z}")) {
[15033]477 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
[15034]478 url = url.replace("{z}", "{zoom}");
[9658]479 }
[9505]480 if (josmUrls.containsKey(url)) {
[15033]481 myprintln("+++ JOSM-URL is not unique: "+url);
[9505]482 } else {
[15033]483 josmUrls.put(url, e);
[7726]484 }
[15033]485 for (ImageryInfo m : e.getMirrors()) {
486 url = getUrlStripped(m);
487 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
488 ReflectionUtils.setObjectsAccessible(origNameField);
[15034]489 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
[9658]490 if (josmUrls.containsKey(url)) {
[15033]491 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
[9658]492 } else {
[15033]493 josmUrls.put(url, m);
494 josmMirrors.put(url, m);
[9658]495 }
496 }
[7726]497 }
[15033]498 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
[7726]499 }
500
[15033]501 void checkInOneButNotTheOther() throws IOException {
502 List<String> le = new LinkedList<>(eliUrls.keySet());
503 List<String> lj = new LinkedList<>(josmUrls.keySet());
[13593]504
[15033]505 List<String> ke = new LinkedList<>(le);
506 for (String url : ke) {
507 if (lj.contains(url)) {
508 le.remove(url);
509 lj.remove(url);
[13593]510 }
511 }
512
[15033]513 if (!le.isEmpty() && !lj.isEmpty()) {
514 ke = new LinkedList<>(le);
515 for (String urle : ke) {
516 JsonObject e = eliUrls.get(urle);
517 String ide = getId(e);
[15034]518 String urlhttps = urle.replace("http:", "https:");
[15033]519 if (lj.contains(urlhttps)) {
520 myprintln("+ Missing https: "+getDescription(e));
521 eliUrls.put(urlhttps, eliUrls.get(urle));
522 eliUrls.remove(urle);
523 le.remove(urle);
524 lj.remove(urlhttps);
525 } else if (isNotBlank(ide)) {
526 List<String> kj = new LinkedList<>(lj);
527 for (String urlj : kj) {
528 ImageryInfo j = josmUrls.get(urlj);
529 String idj = getId(j);
[13714]530
[15033]531 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
532 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
533 le.remove(urle);
534 lj.remove(urlj);
535 // replace key for this entry with JOSM URL
536 eliUrls.remove(urle);
[15034]537 eliUrls.put(urlj, e);
[15033]538 break;
[13593]539 }
540 }
[13517]541 }
[7726]542 }
543 }
[13714]544
[15033]545 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
546 Collections.sort(le);
[13593]547 if (!le.isEmpty()) {
[15033]548 for (String l : le) {
549 myprintln("- " + getDescription(eliUrls.get(l)));
[11412]550 }
[7726]551 }
[15033]552 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
553 Collections.sort(lj);
[13593]554 if (!lj.isEmpty()) {
[15033]555 for (String l : lj) {
556 myprintln("+ " + getDescription(josmUrls.get(l)));
[11412]557 }
[7726]558 }
559 }
[9667]560
[15033]561 void checkCommonEntries() throws IOException {
[15034]562 doSameUrlButDifferentName();
563 doSameUrlButDifferentId();
564 doSameUrlButDifferentType();
565 doSameUrlButDifferentZoomBounds();
566 doSameUrlButDifferentCountryCode();
567 doSameUrlButDifferentQuality();
568 doSameUrlButDifferentDates();
569 doSameUrlButDifferentInformation();
570 doMismatchingShapes();
571 doMismatchingIcons();
572 doMiscellaneousChecks();
573 }
574
575 void doSameUrlButDifferentName() throws IOException {
[15033]576 myprintln("*** Same URL, but different name: ***");
577 for (String url : eliUrls.keySet()) {
578 JsonObject e = eliUrls.get(url);
579 if (!josmUrls.containsKey(url)) continue;
580 ImageryInfo j = josmUrls.get(url);
[15034]581 String ename = getName(e).replace("'", "\u2019");
582 String jname = getName(j).replace("'", "\u2019");
[11951]583 if (!ename.equals(jname)) {
[15033]584 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
[7726]585 }
586 }
[15034]587 }
[9667]588
[15034]589 void doSameUrlButDifferentId() throws IOException {
[15033]590 myprintln("*** Same URL, but different Id: ***");
591 for (String url : eliUrls.keySet()) {
592 JsonObject e = eliUrls.get(url);
593 if (!josmUrls.containsKey(url)) continue;
594 ImageryInfo j = josmUrls.get(url);
595 String ename = getId(e);
596 String jname = getId(j);
597 if (!Objects.equals(ename, jname)) {
598 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
[12126]599 }
[12226]600 }
[15034]601 }
[12126]602
[15034]603 void doSameUrlButDifferentType() throws IOException {
[15033]604 myprintln("*** Same URL, but different type: ***");
605 for (String url : eliUrls.keySet()) {
606 JsonObject e = eliUrls.get(url);
607 if (!josmUrls.containsKey(url)) continue;
608 ImageryInfo j = josmUrls.get(url);
609 if (!Objects.equals(getType(e), getType(j))) {
610 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
[7726]611 }
612 }
[15034]613 }
[9667]614
[15034]615 void doSameUrlButDifferentZoomBounds() throws IOException {
[15033]616 myprintln("*** Same URL, but different zoom bounds: ***");
617 for (String url : eliUrls.keySet()) {
618 JsonObject e = eliUrls.get(url);
619 if (!josmUrls.containsKey(url)) continue;
620 ImageryInfo j = josmUrls.get(url);
[7726]621
[15033]622 Integer eMinZoom = getMinZoom(e);
623 Integer jMinZoom = getMinZoom(j);
[13997]624 /* dont warn for entries copied from the base of the mirror */
[15033]625 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]626 jMinZoom = null;
[15033]627 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
628 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
[7726]629 }
[15033]630 Integer eMaxZoom = getMaxZoom(e);
631 Integer jMaxZoom = getMaxZoom(j);
[13997]632 /* dont warn for entries copied from the base of the mirror */
[15033]633 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]634 jMaxZoom = null;
[15033]635 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
636 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
[7726]637 }
638 }
[15034]639 }
[9667]640
[15034]641 void doSameUrlButDifferentCountryCode() throws IOException {
[15033]642 myprintln("*** Same URL, but different country code: ***");
643 for (String url : eliUrls.keySet()) {
644 JsonObject e = eliUrls.get(url);
645 if (!josmUrls.containsKey(url)) continue;
646 ImageryInfo j = josmUrls.get(url);
647 String cce = getCountryCode(e);
[13931]648 if ("ZZ".equals(cce)) { /* special ELI country code */
[15033]649 cce = null;
[13931]650 }
[15033]651 if (cce != null && !cce.equals(getCountryCode(j))) {
652 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
[7726]653 }
654 }
[15034]655 }
656
657 void doSameUrlButDifferentQuality() throws IOException {
[15033]658 myprintln("*** Same URL, but different quality: ***");
659 for (String url : eliUrls.keySet()) {
660 JsonObject e = eliUrls.get(url);
[9515]661 if (!josmUrls.containsKey(url)) {
[15033]662 String q = getQuality(e);
663 if ("eli-best".equals(q)) {
664 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
[9515]665 }
[15033]666 continue;
[9515]667 }
[15033]668 ImageryInfo j = josmUrls.get(url);
669 if (!Objects.equals(getQuality(e), getQuality(j))) {
670 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
[9505]671 }
[11599]672 }
[15034]673 }
674
675 void doSameUrlButDifferentDates() throws IOException {
[15033]676 myprintln("*** Same URL, but different dates: ***");
677 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
678 for (String url : eliUrls.keySet()) {
679 String ed = getDate(eliUrls.get(url));
680 if (!josmUrls.containsKey(url)) continue;
681 ImageryInfo j = josmUrls.get(url);
682 String jd = getDate(j);
[11612]683 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
[15034]684 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]685 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
[15033]686 String ed2 = ed;
687 Matcher m = pattern.matcher(ed);
688 if (m.matches()) {
689 Calendar cal = Calendar.getInstance();
690 cal.set(Integer.valueOf(m.group(2)),
691 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
692 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
693 cal.add(Calendar.DAY_OF_MONTH, -1);
694 ed2 = m.group(1) + cal.get(Calendar.YEAR);
695 if (m.group(4) != null)
696 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
697 if (m.group(6) != null)
698 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
[11639]699 }
[15034]700 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]701 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
[15033]702 String t = "'"+ed+"'";
[11612]703 if (!ed.equals(ef)) {
[15033]704 t += " or '"+ef+"'";
[11612]705 }
[11666]706 if (jd.isEmpty()) {
[15033]707 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
[11668]708 } else if (!ed.isEmpty()) {
[15033]709 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
710 } else if (!optionNoEli) {
711 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
[11666]712 }
[11573]713 }
[11665]714 }
[15034]715 }
716
717 void doSameUrlButDifferentInformation() throws IOException {
[15033]718 myprintln("*** Same URL, but different information: ***");
719 for (String url : eliUrls.keySet()) {
720 if (!josmUrls.containsKey(url)) continue;
721 JsonObject e = eliUrls.get(url);
722 ImageryInfo j = josmUrls.get(url);
[11981]723
[15082]724 compareDescriptions(e, j);
725 comparePermissionReferenceUrls(e, j);
726 compareAttributionUrls(e, j);
727 compareAttributionTexts(e, j);
728 compareProjections(e, j);
729 compareDefaults(e, j);
730 compareOverlays(e, j);
731 compareNoTileHeaders(e, j);
732 }
733 }
734
735 void compareDescriptions(JsonObject e, ImageryInfo j) throws IOException {
736 String et = getDescriptions(e).getOrDefault("en", "");
737 String jt = getDescriptions(j).getOrDefault("en", "");
738 if (!et.equals(jt)) {
739 if (jt.isEmpty()) {
740 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
741 } else if (!et.isEmpty()) {
742 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
743 } else if (!optionNoEli) {
744 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
[11981]745 }
[15082]746 }
747 }
[11981]748
[15082]749 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) throws IOException {
750 String et = getPermissionReferenceUrl(e);
751 String jt = getPermissionReferenceUrl(j);
752 String jt2 = getTermsOfUseUrl(j);
753 if (isBlank(jt)) jt = jt2;
754 if (!Objects.equals(et, jt)) {
755 if (isBlank(jt)) {
756 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
757 } else if (isNotBlank(et)) {
758 String ethttps = et.replace("http:", "https:");
759 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
760 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
761 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
762 } else {
763 String ja = getAttributionUrl(j);
764 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
765 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
[13578]766 } else {
[15082]767 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13578]768 }
[13551]769 }
[11981]770 }
[15082]771 } else if (!optionNoEli) {
772 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
[11981]773 }
[15082]774 }
775 }
[11981]776
[15082]777 void compareAttributionUrls(JsonObject e, ImageryInfo j) throws IOException {
778 String et = getAttributionUrl(e);
779 String jt = getAttributionUrl(j);
780 if (!Objects.equals(et, jt)) {
781 if (isBlank(jt)) {
782 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
783 } else if (isNotBlank(et)) {
784 String ethttps = et.replace("http:", "https:");
785 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
786 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
787 } else {
788 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[11981]789 }
[15082]790 } else if (!optionNoEli) {
791 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
[11981]792 }
[15082]793 }
794 }
[11981]795
[15082]796 void compareAttributionTexts(JsonObject e, ImageryInfo j) throws IOException {
797 String et = getAttributionText(e);
798 String jt = getAttributionText(j);
799 if (!Objects.equals(et, jt)) {
800 if (isBlank(jt)) {
801 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
802 } else if (isNotBlank(et)) {
803 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
804 } else if (!optionNoEli) {
805 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
[11981]806 }
[15082]807 }
808 }
[11981]809
[15082]810 void compareProjections(JsonObject e, ImageryInfo j) throws IOException {
811 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
812 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
813 if (!Objects.equals(et, jt)) {
814 if (isBlank(jt)) {
815 String t = getType(e);
816 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
817 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
818 } else {
819 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
[11981]820 }
[15082]821 } else if (isNotBlank(et)) {
822 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
823 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
824 } else {
825 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
826 }
827 } else if (!optionNoEli && !"tms".equals(getType(e))) {
828 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
[11981]829 }
[15082]830 }
831 }
[12226]832
[15082]833 void compareDefaults(JsonObject e, ImageryInfo j) throws IOException {
834 boolean ed = getDefault(e);
835 boolean jd = getDefault(j);
836 if (ed != jd) {
837 if (!jd) {
838 myprintln("- Missing JOSM default: "+getDescription(j));
839 } else if (!optionNoEli) {
840 myprintln("+ Missing ELI default: "+getDescription(j));
[12226]841 }
[15082]842 }
843 }
844
845 void compareOverlays(JsonObject e, ImageryInfo j) throws IOException {
846 boolean eo = getOverlay(e);
847 boolean jo = getOverlay(j);
848 if (eo != jo) {
849 if (!jo) {
850 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
851 } else if (!optionNoEli) {
852 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
[13536]853 }
[11981]854 }
[15034]855 }
856
[15082]857 void compareNoTileHeaders(JsonObject e, ImageryInfo j) throws IOException {
858 Map<String, Set<String>> eh = getNoTileHeader(e);
859 Map<String, Set<String>> jh = getNoTileHeader(j);
860 if (!Objects.equals(eh, jh)) {
861 if (jh == null || jh.isEmpty()) {
862 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
863 } else if (eh != null && !eh.isEmpty()) {
864 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
865 } else if (!optionNoEli) {
866 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
867 }
868 }
869 }
870
[15034]871 void doMismatchingShapes() throws IOException {
[15033]872 myprintln("*** Mismatching shapes: ***");
873 for (String url : josmUrls.keySet()) {
874 ImageryInfo j = josmUrls.get(url);
875 int num = 1;
876 for (Shape shape : getShapes(j)) {
877 List<Coordinate> p = shape.getPoints();
[15034]878 if (!p.get(0).equals(p.get(p.size()-1))) {
[15033]879 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
[11410]880 }
[15033]881 for (int nump = 1; nump < p.size(); ++nump) {
882 if (Objects.equals(p.get(nump-1), p.get(nump))) {
883 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
[11964]884 }
885 }
[15033]886 ++num;
[11410]887 }
888 }
[15033]889 for (String url : eliUrls.keySet()) {
890 JsonObject e = eliUrls.get(url);
891 int num = 1;
892 List<Shape> s = null;
[13771]893 try {
[15033]894 s = getShapes(e);
895 for (Shape shape : s) {
896 List<Coordinate> p = shape.getPoints();
[15034]897 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
[15033]898 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
[11964]899 }
[15033]900 for (int nump = 1; nump < p.size(); ++nump) {
901 if (Objects.equals(p.get(nump-1), p.get(nump))) {
902 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
[13771]903 }
904 }
[15033]905 ++num;
[11964]906 }
[15033]907 } catch (IllegalArgumentException err) {
908 String desc = getDescription(e);
909 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
[11410]910 }
[13780]911 if (s == null || !josmUrls.containsKey(url)) {
[15033]912 continue;
[11410]913 }
[15033]914 ImageryInfo j = josmUrls.get(url);
915 List<Shape> js = getShapes(j);
916 if (s.isEmpty() && !js.isEmpty()) {
917 if (!optionNoEli) {
918 myprintln("+ No ELI shape: "+getDescription(j));
[11414]919 }
[15034]920 } else if (js.isEmpty() && !s.isEmpty()) {
[11415]921 // don't report boundary like 5 point shapes as difference
[15033]922 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
923 myprintln("- No JOSM shape: "+getDescription(j));
[11415]924 }
[15034]925 } else if (s.size() != js.size()) {
[15033]926 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
[11413]927 } else {
[15033]928 boolean[] edone = new boolean[s.size()];
929 boolean[] jdone = new boolean[js.size()];
930 for (int enums = 0; enums < s.size(); ++enums) {
931 List<Coordinate> ep = s.get(enums).getPoints();
932 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
933 List<Coordinate> jp = js.get(jnums).getPoints();
934 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]935 boolean err = false;
[15034]936 for (int nump = 0; nump < ep.size() && !err; ++nump) {
[15033]937 Coordinate ept = ep.get(nump);
938 Coordinate jpt = jp.get(nump);
[15034]939 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
[15033]940 err = true;
[11413]941 }
[15034]942 if (!err) {
[15033]943 edone[enums] = true;
944 jdone[jnums] = true;
945 break;
[14576]946 }
[11413]947 }
948 }
[11411]949 }
[15033]950 for (int enums = 0; enums < s.size(); ++enums) {
951 List<Coordinate> ep = s.get(enums).getPoints();
952 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
953 List<Coordinate> jp = js.get(jnums).getPoints();
954 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]955 boolean err = false;
[15033]956 for (int nump = 0; nump < ep.size() && !err; ++nump) {
957 Coordinate ept = ep.get(nump);
958 Coordinate jpt = jp.get(nump);
959 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
960 String numtxt = Integer.toString(enums+1);
961 if (enums != jnums) {
962 numtxt += '/' + Integer.toString(jnums+1);
963 }
964 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
965 break;
[14576]966 }
967 }
[15033]968 edone[enums] = true;
969 jdone[jnums] = true;
970 break;
[14576]971 }
972 }
973 }
[15033]974 for (int enums = 0; enums < s.size(); ++enums) {
975 List<Coordinate> ep = s.get(enums).getPoints();
976 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
977 List<Coordinate> jp = js.get(jnums).getPoints();
978 if (!jdone[jnums]) {
979 String numtxt = Integer.toString(enums+1);
980 if (enums != jnums) {
981 numtxt += '/' + Integer.toString(jnums+1);
982 }
[15034]983 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
984 + getDescription(j));
[15033]985 edone[enums] = true;
986 jdone[jnums] = true;
987 break;
[14576]988 }
989 }
990 }
[11410]991 }
992 }
[15034]993 }
994
995 void doMismatchingIcons() throws IOException {
[15033]996 myprintln("*** Mismatching icons: ***");
997 for (String url : eliUrls.keySet()) {
998 JsonObject e = eliUrls.get(url);
[11420]999 if (!josmUrls.containsKey(url)) {
[15033]1000 continue;
[11420]1001 }
[15033]1002 ImageryInfo j = josmUrls.get(url);
1003 String ij = getIcon(j);
1004 String ie = getIcon(e);
1005 boolean ijok = isNotBlank(ij);
1006 boolean ieok = isNotBlank(ie);
1007 if (ijok && !ieok) {
1008 if (!optionNoEli) {
1009 myprintln("+ No ELI icon: "+getDescription(j));
[11420]1010 }
[15033]1011 } else if (!ijok && ieok) {
1012 myprintln("- No JOSM icon: "+getDescription(j));
1013 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
[14351]1014 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1015 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
[14293]1016 ij.startsWith("data:"))) {
[15034]1017 String iehttps = ie.replace("http:", "https:");
[15033]1018 if (ij.equals(iehttps)) {
1019 myprintln("+ Different icons: "+getDescription(j));
[14294]1020 } else {
[15033]1021 myprintln("* Different icons: "+getDescription(j));
[14294]1022 }
[11420]1023 }
1024 }
[15034]1025 }
1026
1027 void doMiscellaneousChecks() throws IOException {
[15033]1028 myprintln("*** Miscellaneous checks: ***");
1029 Map<String, ImageryInfo> josmIds = new HashMap<>();
1030 Collection<String> all = Projections.getAllProjectionCodes();
[13551]1031 DomainValidator dv = DomainValidator.getInstance();
[15033]1032 for (String url : josmUrls.keySet()) {
1033 ImageryInfo j = josmUrls.get(url);
1034 String id = getId(j);
1035 if ("wms".equals(getType(j))) {
1036 String urlLc = url.toLowerCase(Locale.ENGLISH);
1037 if (getProjections(j).isEmpty()) {
1038 myprintln("* WMS without projections: "+getDescription(j));
[13526]1039 } else {
[15033]1040 List<String> unsupported = new LinkedList<>();
1041 List<String> old = new LinkedList<>();
1042 for (String p : getProjectionsUnstripped(j)) {
1043 if ("CRS:84".equals(p)) {
1044 if (!urlLc.contains("version=1.3")) {
1045 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
[13526]1046 }
[15033]1047 } else if (oldproj.containsKey(p)) {
1048 old.add(p);
1049 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1050 unsupported.add(p);
[13526]1051 }
1052 }
[15033]1053 if (!unsupported.isEmpty()) {
1054 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
[13526]1055 }
[15033]1056 for (String o : old) {
[15034]1057 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1058 + getDescription(j));
[13527]1059 }
[13526]1060 }
[15033]1061 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1062 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1063 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1064 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
[13526]1065 }
[13511]1066 }
[15033]1067 List<String> urls = new LinkedList<>();
1068 if (!"scanex".equals(getType(j))) {
1069 urls.add(url);
[13532]1070 }
[15033]1071 String jt = getPermissionReferenceUrl(j);
1072 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1073 urls.add(jt);
1074 jt = getTermsOfUseUrl(j);
1075 if (isNotBlank(jt))
1076 urls.add(jt);
1077 jt = getAttributionUrl(j);
1078 if (isNotBlank(jt))
1079 urls.add(jt);
1080 jt = getIcon(j);
1081 if (isNotBlank(jt) && !jt.startsWith("data:image/png;base64,"))
1082 urls.add(jt);
1083 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?/.*");
1084 for (String u : urls) {
1085 Matcher m = patternU.matcher(u);
1086 if (!m.matches() || u.matches(".*[ \t]+$")) {
1087 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1088 } else {
[15034]1089 String domain = m.group(1).replaceAll("\\{switch:.*\\}", "x");
[15033]1090 String port = m.group(2);
1091 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1092 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1093 else if (":80".equals(port) || ":443".equals(port)) {
1094 myprintln("* Useless port '"+port+"': "+getDescription(j));
[13554]1095 }
[13551]1096 }
1097 }
1098
[15033]1099 if (josmMirrors.containsKey(url)) {
1100 continue;
[11420]1101 }
[15033]1102 if (isBlank(id)) {
1103 myprintln("* No JOSM-ID: "+getDescription(j));
1104 } else if (josmIds.containsKey(id)) {
1105 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
[11420]1106 } else {
[15033]1107 josmIds.put(id, j);
[11420]1108 }
[15033]1109 String d = getDate(j);
1110 if (isNotBlank(d)) {
1111 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1112 Matcher m = patternD.matcher(d);
1113 if (!m.matches()) {
1114 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
[11572]1115 } else {
1116 try {
[15033]1117 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1118 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1119 if (second.compareTo(first) < 0) {
1120 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
[11572]1121 }
[15034]1122 } catch (Exception e) {
[15033]1123 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
[11572]1124 }
1125 }
[11603]1126 }
[15033]1127 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1128 myprintln("* Attribution link without text: "+getDescription(j));
[12261]1129 }
[15033]1130 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1131 myprintln("* Logo link without image: "+getDescription(j));
[12261]1132 }
[15033]1133 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1134 myprintln("* Terms of Use text without link: "+getDescription(j));
[12261]1135 }
[15033]1136 List<Shape> js = getShapes(j);
1137 if (!js.isEmpty()) {
1138 double minlat = 1000;
1139 double minlon = 1000;
1140 double maxlat = -1000;
1141 double maxlon = -1000;
1142 for (Shape s: js) {
1143 for (Coordinate p: s.getPoints()) {
1144 double lat = p.getLat();
1145 double lon = p.getLon();
[15034]1146 if (lat > maxlat) maxlat = lat;
1147 if (lon > maxlon) maxlon = lon;
1148 if (lat < minlat) minlat = lat;
1149 if (lon < minlon) minlon = lon;
[11422]1150 }
1151 }
[15033]1152 ImageryBounds b = j.getBounds();
1153 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1154 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
[15034]1155 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1156 + getDescription(j));
[11422]1157 }
1158 }
[15033]1159 List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
1160 String cat = getCategory(j);
1161 if (isBlank(cat)) {
1162 myprintln("* No category: "+getDescription(j));
1163 } else if (!knownCategories.contains(cat)) {
1164 myprintln("* Strange category "+cat+": "+getDescription(j));
[13792]1165 }
[11420]1166 }
[7726]1167 }
[9667]1168
[15033]1169 /*
[7726]1170 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1171 */
[15033]1172
[7726]1173 static String getUrl(Object e) {
[15033]1174 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1175 return ((Map<String, JsonObject>) e).get("properties").getString("url");
[7726]1176 }
[15033]1177
[12242]1178 static String getUrlStripped(Object e) {
[15034]1179 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
[12242]1180 }
[15033]1181
[11572]1182 static String getDate(Object e) {
[15033]1183 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1184 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1185 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1186 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
[15034]1187 if (!start.isEmpty() && !end.isEmpty())
[15033]1188 return start+";"+end;
[15034]1189 else if (!start.isEmpty())
[15033]1190 return start+";-";
[15034]1191 else if (!end.isEmpty())
[15033]1192 return "-;"+end;
1193 return "";
[11572]1194 }
[15033]1195
1196 static Date verifyDate(String year, String month, String day) throws ParseException {
1197 String date;
[15034]1198 if (year == null) {
[15033]1199 date = "3000-01-01";
[11854]1200 } else {
[15033]1201 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
[11854]1202 }
[15033]1203 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1204 df.setLenient(false);
1205 return df.parse(date);
[11572]1206 }
[15033]1207
[11420]1208 static String getId(Object e) {
[15033]1209 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1210 return ((Map<String, JsonObject>) e).get("properties").getString("id");
[11420]1211 }
[15033]1212
[7726]1213 static String getName(Object e) {
[15033]1214 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1215 return ((Map<String, JsonObject>) e).get("properties").getString("name");
[7726]1216 }
[15033]1217
1218 static List<ImageryInfo> getMirrors(Object e) {
1219 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1220 return Collections.emptyList();
[11967]1221 }
[15033]1222
1223 static List<String> getProjections(Object e) {
1224 List<String> r = new ArrayList<>();
1225 List<String> u = getProjectionsUnstripped(e);
1226 if (u != null) {
1227 for (String p : u) {
1228 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1229 r.add(p);
[13530]1230 }
1231 }
1232 }
[15033]1233 return r;
[13530]1234 }
[15033]1235
1236 static List<String> getProjectionsUnstripped(Object e) {
1237 List<String> r = null;
[11975]1238 if (e instanceof ImageryInfo) {
[15033]1239 r = ((ImageryInfo) e).getServerProjections();
[11975]1240 } else {
[15033]1241 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1242 if (s != null) {
1243 r = new ArrayList<>();
1244 for (JsonValue p : s.asJsonArray()) {
1245 r.add(((JsonString) p).getString());
[13530]1246 }
[11981]1247 }
[11975]1248 }
[15033]1249 return r != null ? r : Collections.emptyList();
[11975]1250 }
[15033]1251
[11410]1252 static List<Shape> getShapes(Object e) {
1253 if (e instanceof ImageryInfo) {
[15033]1254 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
[15034]1255 if (bounds != null) {
[15033]1256 return bounds.getShapes();
[11411]1257 }
[15033]1258 return Collections.emptyList();
[11410]1259 }
[15033]1260 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1261 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1262 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1263 List<Shape> l = new ArrayList<>();
1264 for (JsonValue shapes: poly) {
1265 Shape s = new Shape();
1266 for (JsonValue point: shapes.asJsonArray()) {
1267 String lon = point.asJsonArray().getJsonNumber(0).toString();
1268 String lat = point.asJsonArray().getJsonNumber(1).toString();
1269 s.addPoint(lat, lon);
[11410]1270 }
[15033]1271 l.add(s);
[11410]1272 }
[15033]1273 return l;
[11410]1274 }
[15033]1275 return Collections.emptyList();
[11410]1276 }
[15033]1277
[7726]1278 static String getType(Object e) {
[15033]1279 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1280 return ((Map<String, JsonObject>) e).get("properties").getString("type");
[7726]1281 }
[15033]1282
[7726]1283 static Integer getMinZoom(Object e) {
1284 if (e instanceof ImageryInfo) {
[15033]1285 int mz = ((ImageryInfo) e).getMinZoom();
1286 return mz == 0 ? null : mz;
[7726]1287 } else {
[15033]1288 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1289 if (num == null) return null;
1290 return num.intValue();
[7726]1291 }
1292 }
[15033]1293
[7726]1294 static Integer getMaxZoom(Object e) {
1295 if (e instanceof ImageryInfo) {
[15033]1296 int mz = ((ImageryInfo) e).getMaxZoom();
1297 return mz == 0 ? null : mz;
[7726]1298 } else {
[15033]1299 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1300 if (num == null) return null;
1301 return num.intValue();
[7726]1302 }
1303 }
[15033]1304
[7726]1305 static String getCountryCode(Object e) {
[15033]1306 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1307 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
[7726]1308 }
[15033]1309
[9505]1310 static String getQuality(Object e) {
[15033]1311 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1312 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1313 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
[9505]1314 }
[15033]1315
1316 static boolean getOverlay(Object e) {
1317 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1318 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1319 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
[13536]1320 }
[15033]1321
[11420]1322 static String getIcon(Object e) {
[15033]1323 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1324 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
[11420]1325 }
[15033]1326
[11975]1327 static String getAttributionText(Object e) {
[15033]1328 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1329 try {
1330 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1331 } catch (NullPointerException ex) {
1332 return null;
1333 }
[11975]1334 }
[15033]1335
[11975]1336 static String getAttributionUrl(Object e) {
[15033]1337 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1338 try {
1339 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1340 } catch (NullPointerException ex) {
1341 return null;
1342 }
[11975]1343 }
[15033]1344
[11975]1345 static String getTermsOfUseText(Object e) {
[15033]1346 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1347 return null;
[11975]1348 }
[15033]1349
[11975]1350 static String getTermsOfUseUrl(Object e) {
[15033]1351 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1352 return null;
[11975]1353 }
[15033]1354
[13792]1355 static String getCategory(Object e) {
1356 if (e instanceof ImageryInfo) {
[15033]1357 return ((ImageryInfo) e).getImageryCategoryOriginalString();
[13792]1358 }
[15033]1359 return null;
[13792]1360 }
[15033]1361
[12261]1362 static String getLogoImage(Object e) {
[15033]1363 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1364 return null;
[12261]1365 }
[15033]1366
[12261]1367 static String getLogoUrl(Object e) {
[15033]1368 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1369 return null;
[12261]1370 }
[15033]1371
[11975]1372 static String getPermissionReferenceUrl(Object e) {
[15033]1373 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1374 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
[11975]1375 }
[15033]1376
[15082]1377 static Map<String, Set<String>> getNoTileHeader(Object e) {
1378 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1379 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1380 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1381 Function.identity(),
1382 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1383 }
1384
[15034]1385 static Map<String, String> getDescriptions(Object e) {
[15082]1386 Map<String, String> res = new HashMap<>();
[11975]1387 if (e instanceof ImageryInfo) {
[15033]1388 String a = ((ImageryInfo) e).getDescription();
1389 if (a != null) res.put("en", a);
[11975]1390 } else {
[15033]1391 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
[15034]1392 if (a != null) res.put("en", a.replaceAll("''", "'"));
[11975]1393 }
[15033]1394 return res;
[11975]1395 }
[15033]1396
1397 static boolean getValidGeoreference(Object e) {
1398 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1399 return false;
[11975]1400 }
[15033]1401
1402 static boolean getDefault(Object e) {
1403 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1404 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
[12226]1405 }
[15033]1406
[7726]1407 String getDescription(Object o) {
[15033]1408 String url = getUrl(o);
1409 String cc = getCountryCode(o);
[7726]1410 if (cc == null) {
[15033]1411 ImageryInfo j = josmUrls.get(url);
1412 if (j != null) cc = getCountryCode(j);
[7726]1413 if (cc == null) {
[15033]1414 JsonObject e = eliUrls.get(url);
1415 if (e != null) cc = getCountryCode(e);
[7726]1416 }
1417 }
1418 if (cc == null) {
[15033]1419 cc = "";
[7726]1420 } else {
[15033]1421 cc = "["+cc+"] ";
[7726]1422 }
[15033]1423 String d = cc + getName(o) + " - " + getUrl(o);
1424 if (optionShorten) {
1425 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
[7726]1426 }
[15033]1427 return d;
[7726]1428 }
1429}
Note: See TracBrowser for help on using the repository browser.