source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15304

Last change on this file since 15304 was 15163, checked in by Don-vip, 7 years ago

fix URL parsing errors

  • Property svn:eol-style set to native
File size: 62.9 KB
RevLine 
[7726]1// License: GPL. For details, see LICENSE file.
[15034]2import static java.nio.charset.StandardCharsets.UTF_8;
[15033]3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
[15034]9import java.io.OutputStream;
[15033]10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
[15163]12import java.net.MalformedURLException;
13import java.net.URL;
[15034]14import java.nio.file.Files;
15import java.nio.file.Paths;
[15033]16import java.text.DecimalFormat;
17import java.text.ParseException;
18import java.text.SimpleDateFormat;
19import java.util.ArrayList;
20import java.util.Arrays;
21import java.util.Calendar;
22import java.util.Collection;
23import java.util.Collections;
24import java.util.Date;
25import java.util.HashMap;
26import java.util.LinkedList;
27import java.util.List;
28import java.util.Locale;
29import java.util.Map;
30import java.util.Map.Entry;
31import java.util.Objects;
[15082]32import java.util.Set;
33import java.util.function.Function;
[15033]34import java.util.regex.Matcher;
35import java.util.regex.Pattern;
36import java.util.stream.Collectors;
37
38import javax.json.Json;
39import javax.json.JsonArray;
40import javax.json.JsonNumber;
41import javax.json.JsonObject;
42import javax.json.JsonReader;
43import javax.json.JsonString;
44import javax.json.JsonValue;
45
46import org.openstreetmap.gui.jmapviewer.Coordinate;
47import org.openstreetmap.josm.data.Preferences;
48import org.openstreetmap.josm.data.imagery.ImageryInfo;
49import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
50import org.openstreetmap.josm.data.imagery.Shape;
51import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
52import org.openstreetmap.josm.data.projection.Projections;
53import org.openstreetmap.josm.data.validation.routines.DomainValidator;
54import org.openstreetmap.josm.io.imagery.ImageryReader;
55import org.openstreetmap.josm.spi.preferences.Config;
[15034]56import org.openstreetmap.josm.tools.Logging;
[15033]57import org.openstreetmap.josm.tools.OptionParser;
58import org.openstreetmap.josm.tools.OptionParser.OptionCount;
59import org.openstreetmap.josm.tools.ReflectionUtils;
60import org.xml.sax.SAXException;
61
[7726]62/**
[11854]63 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
[7726]64 * The goal is to keep both lists in sync.
65 *
[11854]66 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
[11694]67 * provides also a version in the JOSM format, but the GEOJSON is the original source
[7726]68 * format, so we read that.
69 *
70 * How to run:
71 * -----------
72 *
73 * Main JOSM binary needs to be in classpath, e.g.
74 *
[15033]75 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
[9667]76 *
[7726]77 * Add option "-h" to show the available command line flags.
78 */
[15037]79@SuppressWarnings("unchecked")
[15033]80public class SyncEditorLayerIndex {
[14019]81
[15034]82 private static final int MAXLEN = 140;
83
[15033]84 private List<ImageryInfo> josmEntries;
85 private JsonArray eliEntries;
[7726]86
[15034]87 private final Map<String, JsonObject> eliUrls = new HashMap<>();
88 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
89 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
90 private static final Map<String, String> oldproj = new HashMap<>();
91 private static final List<String> ignoreproj = new LinkedList<>();
[7726]92
[15033]93 private static String eliInputFile = "imagery_eli.geojson";
94 private static String josmInputFile = "imagery_josm.imagery.xml";
95 private static String ignoreInputFile = "imagery_josm.ignores.txt";
[15034]96 private static OutputStream outputFile;
97 private static OutputStreamWriter outputStream;
[15033]98 private static String optionOutput;
99 private static boolean optionShorten;
100 private static boolean optionNoSkip;
101 private static boolean optionXhtmlBody;
102 private static boolean optionXhtml;
103 private static String optionEliXml;
104 private static String optionJosmXml;
105 private static String optionEncoding;
106 private static boolean optionNoEli;
107 private Map<String, String> skip = new HashMap<>();
[7726]108
109 /**
110 * Main method.
[15033]111 * @param args program arguments
112 * @throws IOException if any I/O error occurs
113 * @throws ReflectiveOperationException if any reflective operation error occurs
114 * @throws SAXException if any SAX error occurs
[7726]115 */
[15033]116 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
117 Locale.setDefault(Locale.ROOT);
[15034]118 parseCommandLineArguments(args);
[15033]119 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
120 Config.setPreferencesInstance(pref);
121 pref.init(false);
122 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
123 script.setupProj();
124 script.loadSkip();
125 script.start();
126 script.loadJosmEntries();
127 if (optionJosmXml != null) {
[15034]128 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
129 script.printentries(script.josmEntries, stream);
130 }
[11964]131 }
[15033]132 script.loadELIEntries();
133 if (optionEliXml != null) {
[15034]134 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
135 script.printentries(script.eliEntries, stream);
136 }
[11964]137 }
[15033]138 script.checkInOneButNotTheOther();
139 script.checkCommonEntries();
140 script.end();
141 if (outputStream != null) {
142 outputStream.close();
[9505]143 }
[15033]144 if (outputFile != null) {
145 outputFile.close();
[9505]146 }
[7726]147 }
[9653]148
[7726]149 /**
[15033]150 * Displays help on the console
151 */
152 private static void showHelp() {
153 System.out.println(getHelp());
154 System.exit(0);
155 }
156
157 static String getHelp() {
158 return "usage: java -cp build SyncEditorLayerIndex\n" +
159 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
[15034]160 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
161 "Default is imagery_eli.geojson (current directory).\n" +
[15033]162 "-h,--help show this help\n" +
163 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
[15034]164 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
165 "Default is imagery_josm.imagery.xml (current directory).\n" +
[15033]166 "-m,--noeli don't show output for ELI problems\n" +
167 "-n,--noskip don't skip known entries\n" +
168 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
169 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
170 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
171 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
172 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
173 "-X,--xhtml create XHTML for display in a web page\n";
174 }
175
176 /**
[7726]177 * Parse command line arguments.
[15033]178 * @param args program arguments
[15034]179 * @throws IOException in case of I/O error
[7726]180 */
[15034]181 static void parseCommandLineArguments(String[] args) throws IOException {
[15033]182 new OptionParser("JOSM/ELI synchronization script")
183 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
184 .addShortAlias("help", "h")
185 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
186 .addShortAlias("output", "o")
187 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
188 .addShortAlias("eli_input", "e")
189 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
190 .addShortAlias("josm_input", "j")
191 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
192 .addShortAlias("ignore_input", "i")
193 .addFlagParameter("shorten", () -> optionShorten = true)
194 .addShortAlias("shorten", "s")
195 .addFlagParameter("noskip", () -> optionNoSkip = true)
196 .addShortAlias("noskip", "n")
197 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
198 .addShortAlias("xhtmlbody", "x")
199 .addFlagParameter("xhtml", () -> optionXhtml = true)
200 .addShortAlias("xhtml", "X")
201 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
202 .addShortAlias("elixml", "p")
203 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
204 .addShortAlias("josmxml", "q")
205 .addFlagParameter("noeli", () -> optionNoEli = true)
206 .addShortAlias("noeli", "m")
207 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
208 .addShortAlias("encoding", "c")
209 .parseOptionsOrExit(Arrays.asList(args));
[7726]210
[15034]211 if (optionOutput != null && !"-".equals(optionOutput)) {
212 outputFile = Files.newOutputStream(Paths.get(optionOutput));
[15033]213 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
214 } else if (optionEncoding != null) {
215 outputStream = new OutputStreamWriter(System.out, optionEncoding);
[7726]216 }
217 }
218
[13530]219 void setupProj() {
[15033]220 oldproj.put("EPSG:3359", "EPSG:3404");
221 oldproj.put("EPSG:3785", "EPSG:3857");
222 oldproj.put("EPSG:31297", "EPGS:31287");
223 oldproj.put("EPSG:31464", "EPSG:31468");
224 oldproj.put("EPSG:54004", "EPSG:3857");
225 oldproj.put("EPSG:102100", "EPSG:3857");
226 oldproj.put("EPSG:102113", "EPSG:3857");
227 oldproj.put("EPSG:900913", "EPGS:3857");
228 ignoreproj.add("EPSG:4267");
229 ignoreproj.add("EPSG:5221");
230 ignoreproj.add("EPSG:5514");
231 ignoreproj.add("EPSG:32019");
232 ignoreproj.add("EPSG:102066");
233 ignoreproj.add("EPSG:102067");
234 ignoreproj.add("EPSG:102685");
235 ignoreproj.add("EPSG:102711");
[13530]236 }
237
[15033]238 void loadSkip() throws IOException {
239 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
[15034]240 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
[15033]241 String line;
[11238]242
[15033]243 while ((line = fr.readLine()) != null) {
244 Matcher res = pattern.matcher(line);
245 if (res.matches()) {
246 if ("Ignore".equals(res.group(1))) {
247 skip.put(res.group(2), "green");
248 } else {
249 skip.put(res.group(2), "darkgoldenrod");
250 }
[11238]251 }
252 }
[11234]253 }
[11238]254 }
[9653]255
[15033]256 void myprintlnfinal(String s) throws IOException {
257 if (outputStream != null) {
258 outputStream.write(s + System.getProperty("line.separator"));
[9658]259 } else {
[15033]260 System.out.println(s);
[9658]261 }
262 }
263
[15033]264 void myprintln(String s) throws IOException {
265 if (skip.containsKey(s)) {
266 String color = skip.get(s);
267 skip.remove(s);
268 if (optionXhtmlBody || optionXhtml) {
[15034]269 s = "<pre style=\"margin:3px;color:"+color+"\">"
270 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9658]271 }
[15033]272 if (!optionNoSkip) {
273 return;
[9662]274 }
[15034]275 } else if (optionXhtmlBody || optionXhtml) {
[15033]276 String color =
277 s.startsWith("***") ? "black" :
278 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
279 (s.startsWith("#") ? "indigo" :
280 (s.startsWith("!") ? "orange" : "red")));
[15034]281 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9505]282 }
[15033]283 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
284 return;
[11965]285 }
[15033]286 myprintlnfinal(s);
[9658]287 }
288
[15033]289 void start() throws IOException {
290 if (optionXhtml) {
[15034]291 myprintlnfinal(
292 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
293 myprintlnfinal(
294 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
295 "<title>JOSM - ELI differences</title></head><body>\n");
[9505]296 }
297 }
[9653]298
[15033]299 void end() throws IOException {
300 for (String s : skip.keySet()) {
301 myprintln("+++ Obsolete skip entry: " + s);
[9658]302 }
[15033]303 if (optionXhtml) {
304 myprintlnfinal("</body></html>\n");
[9658]305 }
306 }
307
[15033]308 void loadELIEntries() throws IOException {
[15034]309 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
[15033]310 eliEntries = jr.readObject().getJsonArray("features");
311 }
[9653]312
[15033]313 for (JsonValue e : eliEntries) {
314 String url = getUrlStripped(e);
[9653]315 if (url.contains("{z}")) {
[15033]316 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
[15034]317 url = url.replace("{z}", "{zoom}");
[9653]318 }
[11582]319 if (eliUrls.containsKey(url)) {
[15033]320 myprintln("+++ ELI-URL is not unique: "+url);
[9505]321 } else {
[15033]322 eliUrls.put(url, e.asJsonObject());
[9505]323 }
[15033]324 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
325 if (s != null) {
326 String urlLc = url.toLowerCase(Locale.ENGLISH);
327 List<String> old = new LinkedList<>();
328 for (JsonValue p : s) {
329 String proj = ((JsonString) p).getString();
330 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
331 old.add(proj);
[13530]332 }
333 }
[15033]334 if (!old.isEmpty()) {
335 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
[13530]336 }
337 }
[7726]338 }
[15033]339 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
[7726]340 }
[15033]341
342 String cdata(String s) {
343 return cdata(s, false);
[11968]344 }
[7726]345
[15033]346 String cdata(String s, boolean escape) {
347 if (escape) {
348 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
349 } else if (s.matches("[<>&]"))
350 return "<![CDATA["+s+"]]>";
351 return s;
352 }
353
354 String maininfo(Object entry, String offset) {
355 String t = getType(entry);
356 String res = offset + "<type>"+t+"</type>\n";
357 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
358 if (getMinZoom(entry) != null)
359 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
360 if (getMaxZoom(entry) != null)
361 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
362 if ("wms".equals(t)) {
363 List<String> p = getProjections(entry);
364 if (p != null) {
365 res += offset + "<projections>\n";
[15034]366 for (String c : p) {
[15033]367 res += offset + " <code>"+c+"</code>\n";
[15034]368 }
[15033]369 res += offset + "</projections>\n";
[11975]370 }
[11967]371 }
[15033]372 return res;
[11967]373 }
[12061]374
[15033]375 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
376 DecimalFormat df = new DecimalFormat("#.#######");
377 df.setRoundingMode(java.math.RoundingMode.CEILING);
378 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
379 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
380 for (Object e : entries) {
[13536]381 stream.write(" <entry"
[15034]382 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
383 + (getOverlay(e) ? " overlay=\"true\"" : "")
[15033]384 + ">\n");
385 String t;
386 if (isNotBlank(t = getName(e)))
387 stream.write(" <name>"+cdata(t, true)+"</name>\n");
388 if (isNotBlank(t = getId(e)))
389 stream.write(" <id>"+t+"</id>\n");
390 if (isNotBlank(t = getCategory(e)))
391 stream.write(" <category>"+t+"</category>\n");
392 if (isNotBlank(t = getDate(e)))
393 stream.write(" <date>"+t+"</date>\n");
394 if (isNotBlank(t = getCountryCode(e)))
395 stream.write(" <country-code>"+t+"</country-code>\n");
396 if ((getDefault(e)))
397 stream.write(" <default>true</default>\n");
398 stream.write(maininfo(e, " "));
399 if (isNotBlank(t = getAttributionText(e)))
400 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
401 if (isNotBlank(t = getAttributionUrl(e)))
402 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
403 if (isNotBlank(t = getLogoImage(e)))
404 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
405 if (isNotBlank(t = getLogoUrl(e)))
406 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
407 if (isNotBlank(t = getTermsOfUseText(e)))
408 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
409 if (isNotBlank(t = getTermsOfUseUrl(e)))
410 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
411 if (isNotBlank(t = getPermissionReferenceUrl(e)))
412 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
413 if ((getValidGeoreference(e)))
414 stream.write(" <valid-georeference>true</valid-georeference>\n");
415 if (isNotBlank(t = getIcon(e)))
416 stream.write(" <icon>"+cdata(t)+"</icon>\n");
417 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
418 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
[11975]419 }
[15033]420 for (ImageryInfo m : getMirrors(e)) {
421 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
[11967]422 }
[15033]423 double minlat = 1000;
424 double minlon = 1000;
425 double maxlat = -1000;
426 double maxlon = -1000;
427 String shapes = "";
428 String sep = "\n ";
[13771]429 try {
[15033]430 for (Shape s: getShapes(e)) {
431 shapes += " <shape>";
432 int i = 0;
433 for (Coordinate p: s.getPoints()) {
434 double lat = p.getLat();
435 double lon = p.getLon();
436 if (lat > maxlat) maxlat = lat;
437 if (lon > maxlon) maxlon = lon;
438 if (lat < minlat) minlat = lat;
439 if (lon < minlon) minlon = lon;
[15034]440 if ((i++ % 3) == 0) {
[15033]441 shapes += sep + " ";
[13771]442 }
[15033]443 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
[11964]444 }
[15033]445 shapes += sep + "</shape>\n";
[11964]446 }
[15034]447 } catch (IllegalArgumentException ignored) {
448 Logging.trace(ignored);
[11964]449 }
[15033]450 if (!shapes.isEmpty()) {
[15034]451 stream.write(" <bounds min-lat='"+df.format(minlat)
452 +"' min-lon='"+df.format(minlon)
453 +"' max-lat='"+df.format(maxlat)
454 +"' max-lon='"+df.format(maxlon)+"'>\n");
[15033]455 stream.write(shapes + " </bounds>\n");
[11964]456 }
[15033]457 stream.write(" </entry>\n");
[11964]458 }
[15033]459 stream.write("</imagery>\n");
460 stream.close();
[11964]461 }
462
[15033]463 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
464 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
465 josmEntries = reader.parse();
466 }
[9667]467
[15033]468 for (ImageryInfo e : josmEntries) {
469 if (isBlank(getUrl(e))) {
470 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
471 continue;
[14554]472 }
[15033]473 if (isBlank(getName(e))) {
474 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
475 continue;
[14554]476 }
[15033]477 String url = getUrlStripped(e);
[9658]478 if (url.contains("{z}")) {
[15033]479 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
[15034]480 url = url.replace("{z}", "{zoom}");
[9658]481 }
[9505]482 if (josmUrls.containsKey(url)) {
[15033]483 myprintln("+++ JOSM-URL is not unique: "+url);
[9505]484 } else {
[15033]485 josmUrls.put(url, e);
[7726]486 }
[15033]487 for (ImageryInfo m : e.getMirrors()) {
488 url = getUrlStripped(m);
489 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
490 ReflectionUtils.setObjectsAccessible(origNameField);
[15034]491 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
[9658]492 if (josmUrls.containsKey(url)) {
[15033]493 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
[9658]494 } else {
[15033]495 josmUrls.put(url, m);
496 josmMirrors.put(url, m);
[9658]497 }
498 }
[7726]499 }
[15033]500 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
[7726]501 }
502
[15033]503 void checkInOneButNotTheOther() throws IOException {
504 List<String> le = new LinkedList<>(eliUrls.keySet());
505 List<String> lj = new LinkedList<>(josmUrls.keySet());
[13593]506
[15033]507 List<String> ke = new LinkedList<>(le);
508 for (String url : ke) {
509 if (lj.contains(url)) {
510 le.remove(url);
511 lj.remove(url);
[13593]512 }
513 }
514
[15033]515 if (!le.isEmpty() && !lj.isEmpty()) {
516 ke = new LinkedList<>(le);
517 for (String urle : ke) {
518 JsonObject e = eliUrls.get(urle);
519 String ide = getId(e);
[15034]520 String urlhttps = urle.replace("http:", "https:");
[15033]521 if (lj.contains(urlhttps)) {
522 myprintln("+ Missing https: "+getDescription(e));
523 eliUrls.put(urlhttps, eliUrls.get(urle));
524 eliUrls.remove(urle);
525 le.remove(urle);
526 lj.remove(urlhttps);
527 } else if (isNotBlank(ide)) {
528 List<String> kj = new LinkedList<>(lj);
529 for (String urlj : kj) {
530 ImageryInfo j = josmUrls.get(urlj);
531 String idj = getId(j);
[13714]532
[15033]533 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
534 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
535 le.remove(urle);
536 lj.remove(urlj);
537 // replace key for this entry with JOSM URL
538 eliUrls.remove(urle);
[15034]539 eliUrls.put(urlj, e);
[15033]540 break;
[13593]541 }
542 }
[13517]543 }
[7726]544 }
545 }
[13714]546
[15033]547 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
548 Collections.sort(le);
[13593]549 if (!le.isEmpty()) {
[15033]550 for (String l : le) {
551 myprintln("- " + getDescription(eliUrls.get(l)));
[11412]552 }
[7726]553 }
[15033]554 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
555 Collections.sort(lj);
[13593]556 if (!lj.isEmpty()) {
[15033]557 for (String l : lj) {
558 myprintln("+ " + getDescription(josmUrls.get(l)));
[11412]559 }
[7726]560 }
561 }
[9667]562
[15033]563 void checkCommonEntries() throws IOException {
[15034]564 doSameUrlButDifferentName();
565 doSameUrlButDifferentId();
566 doSameUrlButDifferentType();
567 doSameUrlButDifferentZoomBounds();
568 doSameUrlButDifferentCountryCode();
569 doSameUrlButDifferentQuality();
570 doSameUrlButDifferentDates();
571 doSameUrlButDifferentInformation();
572 doMismatchingShapes();
573 doMismatchingIcons();
574 doMiscellaneousChecks();
575 }
576
577 void doSameUrlButDifferentName() throws IOException {
[15033]578 myprintln("*** Same URL, but different name: ***");
579 for (String url : eliUrls.keySet()) {
580 JsonObject e = eliUrls.get(url);
581 if (!josmUrls.containsKey(url)) continue;
582 ImageryInfo j = josmUrls.get(url);
[15034]583 String ename = getName(e).replace("'", "\u2019");
584 String jname = getName(j).replace("'", "\u2019");
[11951]585 if (!ename.equals(jname)) {
[15033]586 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
[7726]587 }
588 }
[15034]589 }
[9667]590
[15034]591 void doSameUrlButDifferentId() throws IOException {
[15033]592 myprintln("*** Same URL, but different Id: ***");
593 for (String url : eliUrls.keySet()) {
594 JsonObject e = eliUrls.get(url);
595 if (!josmUrls.containsKey(url)) continue;
596 ImageryInfo j = josmUrls.get(url);
597 String ename = getId(e);
598 String jname = getId(j);
599 if (!Objects.equals(ename, jname)) {
600 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
[12126]601 }
[12226]602 }
[15034]603 }
[12126]604
[15034]605 void doSameUrlButDifferentType() throws IOException {
[15033]606 myprintln("*** Same URL, but different type: ***");
607 for (String url : eliUrls.keySet()) {
608 JsonObject e = eliUrls.get(url);
609 if (!josmUrls.containsKey(url)) continue;
610 ImageryInfo j = josmUrls.get(url);
611 if (!Objects.equals(getType(e), getType(j))) {
612 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
[7726]613 }
614 }
[15034]615 }
[9667]616
[15034]617 void doSameUrlButDifferentZoomBounds() throws IOException {
[15033]618 myprintln("*** Same URL, but different zoom bounds: ***");
619 for (String url : eliUrls.keySet()) {
620 JsonObject e = eliUrls.get(url);
621 if (!josmUrls.containsKey(url)) continue;
622 ImageryInfo j = josmUrls.get(url);
[7726]623
[15033]624 Integer eMinZoom = getMinZoom(e);
625 Integer jMinZoom = getMinZoom(j);
[13997]626 /* dont warn for entries copied from the base of the mirror */
[15033]627 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]628 jMinZoom = null;
[15033]629 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
630 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
[7726]631 }
[15033]632 Integer eMaxZoom = getMaxZoom(e);
633 Integer jMaxZoom = getMaxZoom(j);
[13997]634 /* dont warn for entries copied from the base of the mirror */
[15033]635 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]636 jMaxZoom = null;
[15033]637 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
638 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
[7726]639 }
640 }
[15034]641 }
[9667]642
[15034]643 void doSameUrlButDifferentCountryCode() throws IOException {
[15033]644 myprintln("*** Same URL, but different country code: ***");
645 for (String url : eliUrls.keySet()) {
646 JsonObject e = eliUrls.get(url);
647 if (!josmUrls.containsKey(url)) continue;
648 ImageryInfo j = josmUrls.get(url);
649 String cce = getCountryCode(e);
[13931]650 if ("ZZ".equals(cce)) { /* special ELI country code */
[15033]651 cce = null;
[13931]652 }
[15033]653 if (cce != null && !cce.equals(getCountryCode(j))) {
654 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
[7726]655 }
656 }
[15034]657 }
658
659 void doSameUrlButDifferentQuality() throws IOException {
[15033]660 myprintln("*** Same URL, but different quality: ***");
661 for (String url : eliUrls.keySet()) {
662 JsonObject e = eliUrls.get(url);
[9515]663 if (!josmUrls.containsKey(url)) {
[15033]664 String q = getQuality(e);
665 if ("eli-best".equals(q)) {
666 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
[9515]667 }
[15033]668 continue;
[9515]669 }
[15033]670 ImageryInfo j = josmUrls.get(url);
671 if (!Objects.equals(getQuality(e), getQuality(j))) {
672 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
[9505]673 }
[11599]674 }
[15034]675 }
676
677 void doSameUrlButDifferentDates() throws IOException {
[15033]678 myprintln("*** Same URL, but different dates: ***");
679 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
680 for (String url : eliUrls.keySet()) {
681 String ed = getDate(eliUrls.get(url));
682 if (!josmUrls.containsKey(url)) continue;
683 ImageryInfo j = josmUrls.get(url);
684 String jd = getDate(j);
[11612]685 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
[15034]686 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]687 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
[15033]688 String ed2 = ed;
689 Matcher m = pattern.matcher(ed);
690 if (m.matches()) {
691 Calendar cal = Calendar.getInstance();
692 cal.set(Integer.valueOf(m.group(2)),
693 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
694 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
695 cal.add(Calendar.DAY_OF_MONTH, -1);
696 ed2 = m.group(1) + cal.get(Calendar.YEAR);
697 if (m.group(4) != null)
698 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
699 if (m.group(6) != null)
700 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
[11639]701 }
[15034]702 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]703 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
[15033]704 String t = "'"+ed+"'";
[11612]705 if (!ed.equals(ef)) {
[15033]706 t += " or '"+ef+"'";
[11612]707 }
[11666]708 if (jd.isEmpty()) {
[15033]709 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
[11668]710 } else if (!ed.isEmpty()) {
[15033]711 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
712 } else if (!optionNoEli) {
713 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
[11666]714 }
[11573]715 }
[11665]716 }
[15034]717 }
718
719 void doSameUrlButDifferentInformation() throws IOException {
[15033]720 myprintln("*** Same URL, but different information: ***");
721 for (String url : eliUrls.keySet()) {
722 if (!josmUrls.containsKey(url)) continue;
723 JsonObject e = eliUrls.get(url);
724 ImageryInfo j = josmUrls.get(url);
[11981]725
[15082]726 compareDescriptions(e, j);
727 comparePermissionReferenceUrls(e, j);
728 compareAttributionUrls(e, j);
729 compareAttributionTexts(e, j);
730 compareProjections(e, j);
731 compareDefaults(e, j);
732 compareOverlays(e, j);
733 compareNoTileHeaders(e, j);
734 }
735 }
736
737 void compareDescriptions(JsonObject e, ImageryInfo j) throws IOException {
738 String et = getDescriptions(e).getOrDefault("en", "");
739 String jt = getDescriptions(j).getOrDefault("en", "");
740 if (!et.equals(jt)) {
741 if (jt.isEmpty()) {
742 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
743 } else if (!et.isEmpty()) {
744 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
745 } else if (!optionNoEli) {
746 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
[11981]747 }
[15082]748 }
749 }
[11981]750
[15082]751 void comparePermissionReferenceUrls(JsonObject e, ImageryInfo j) throws IOException {
752 String et = getPermissionReferenceUrl(e);
753 String jt = getPermissionReferenceUrl(j);
754 String jt2 = getTermsOfUseUrl(j);
755 if (isBlank(jt)) jt = jt2;
756 if (!Objects.equals(et, jt)) {
757 if (isBlank(jt)) {
758 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
759 } else if (isNotBlank(et)) {
760 String ethttps = et.replace("http:", "https:");
761 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
762 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
763 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
764 } else {
765 String ja = getAttributionUrl(j);
766 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
767 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
[13578]768 } else {
[15082]769 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13578]770 }
[13551]771 }
[11981]772 }
[15082]773 } else if (!optionNoEli) {
774 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
[11981]775 }
[15082]776 }
777 }
[11981]778
[15082]779 void compareAttributionUrls(JsonObject e, ImageryInfo j) throws IOException {
780 String et = getAttributionUrl(e);
781 String jt = getAttributionUrl(j);
782 if (!Objects.equals(et, jt)) {
783 if (isBlank(jt)) {
784 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
785 } else if (isNotBlank(et)) {
786 String ethttps = et.replace("http:", "https:");
787 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
788 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
789 } else {
790 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[11981]791 }
[15082]792 } else if (!optionNoEli) {
793 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
[11981]794 }
[15082]795 }
796 }
[11981]797
[15082]798 void compareAttributionTexts(JsonObject e, ImageryInfo j) throws IOException {
799 String et = getAttributionText(e);
800 String jt = getAttributionText(j);
801 if (!Objects.equals(et, jt)) {
802 if (isBlank(jt)) {
803 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
804 } else if (isNotBlank(et)) {
805 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
806 } else if (!optionNoEli) {
807 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
[11981]808 }
[15082]809 }
810 }
[11981]811
[15082]812 void compareProjections(JsonObject e, ImageryInfo j) throws IOException {
813 String et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
814 String jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
815 if (!Objects.equals(et, jt)) {
816 if (isBlank(jt)) {
817 String t = getType(e);
818 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
819 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
820 } else {
821 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
[11981]822 }
[15082]823 } else if (isNotBlank(et)) {
824 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
825 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
826 } else {
827 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
828 }
829 } else if (!optionNoEli && !"tms".equals(getType(e))) {
830 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
[11981]831 }
[15082]832 }
833 }
[12226]834
[15082]835 void compareDefaults(JsonObject e, ImageryInfo j) throws IOException {
836 boolean ed = getDefault(e);
837 boolean jd = getDefault(j);
838 if (ed != jd) {
839 if (!jd) {
840 myprintln("- Missing JOSM default: "+getDescription(j));
841 } else if (!optionNoEli) {
842 myprintln("+ Missing ELI default: "+getDescription(j));
[12226]843 }
[15082]844 }
845 }
846
847 void compareOverlays(JsonObject e, ImageryInfo j) throws IOException {
848 boolean eo = getOverlay(e);
849 boolean jo = getOverlay(j);
850 if (eo != jo) {
851 if (!jo) {
852 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
853 } else if (!optionNoEli) {
854 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
[13536]855 }
[11981]856 }
[15034]857 }
858
[15082]859 void compareNoTileHeaders(JsonObject e, ImageryInfo j) throws IOException {
860 Map<String, Set<String>> eh = getNoTileHeader(e);
861 Map<String, Set<String>> jh = getNoTileHeader(j);
862 if (!Objects.equals(eh, jh)) {
863 if (jh == null || jh.isEmpty()) {
864 myprintln("- Missing JOSM no tile headers ("+eh+"): "+getDescription(j));
865 } else if (eh != null && !eh.isEmpty()) {
866 myprintln("* No tile headers differ ('"+eh+"' != '"+jh+"'): "+getDescription(j));
867 } else if (!optionNoEli) {
868 myprintln("+ Missing ELI no tile headers ('"+jh+"'): "+getDescription(j));
869 }
870 }
871 }
872
[15034]873 void doMismatchingShapes() throws IOException {
[15033]874 myprintln("*** Mismatching shapes: ***");
875 for (String url : josmUrls.keySet()) {
876 ImageryInfo j = josmUrls.get(url);
877 int num = 1;
878 for (Shape shape : getShapes(j)) {
879 List<Coordinate> p = shape.getPoints();
[15034]880 if (!p.get(0).equals(p.get(p.size()-1))) {
[15033]881 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
[11410]882 }
[15033]883 for (int nump = 1; nump < p.size(); ++nump) {
884 if (Objects.equals(p.get(nump-1), p.get(nump))) {
885 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
[11964]886 }
887 }
[15033]888 ++num;
[11410]889 }
890 }
[15033]891 for (String url : eliUrls.keySet()) {
892 JsonObject e = eliUrls.get(url);
893 int num = 1;
894 List<Shape> s = null;
[13771]895 try {
[15033]896 s = getShapes(e);
897 for (Shape shape : s) {
898 List<Coordinate> p = shape.getPoints();
[15034]899 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
[15033]900 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
[11964]901 }
[15033]902 for (int nump = 1; nump < p.size(); ++nump) {
903 if (Objects.equals(p.get(nump-1), p.get(nump))) {
904 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
[13771]905 }
906 }
[15033]907 ++num;
[11964]908 }
[15033]909 } catch (IllegalArgumentException err) {
910 String desc = getDescription(e);
911 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
[11410]912 }
[13780]913 if (s == null || !josmUrls.containsKey(url)) {
[15033]914 continue;
[11410]915 }
[15033]916 ImageryInfo j = josmUrls.get(url);
917 List<Shape> js = getShapes(j);
918 if (s.isEmpty() && !js.isEmpty()) {
919 if (!optionNoEli) {
920 myprintln("+ No ELI shape: "+getDescription(j));
[11414]921 }
[15034]922 } else if (js.isEmpty() && !s.isEmpty()) {
[11415]923 // don't report boundary like 5 point shapes as difference
[15033]924 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
925 myprintln("- No JOSM shape: "+getDescription(j));
[11415]926 }
[15034]927 } else if (s.size() != js.size()) {
[15033]928 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
[11413]929 } else {
[15033]930 boolean[] edone = new boolean[s.size()];
931 boolean[] jdone = new boolean[js.size()];
932 for (int enums = 0; enums < s.size(); ++enums) {
933 List<Coordinate> ep = s.get(enums).getPoints();
934 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
935 List<Coordinate> jp = js.get(jnums).getPoints();
936 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]937 boolean err = false;
[15034]938 for (int nump = 0; nump < ep.size() && !err; ++nump) {
[15033]939 Coordinate ept = ep.get(nump);
940 Coordinate jpt = jp.get(nump);
[15034]941 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
[15033]942 err = true;
[11413]943 }
[15034]944 if (!err) {
[15033]945 edone[enums] = true;
946 jdone[jnums] = true;
947 break;
[14576]948 }
[11413]949 }
950 }
[11411]951 }
[15033]952 for (int enums = 0; enums < s.size(); ++enums) {
953 List<Coordinate> ep = s.get(enums).getPoints();
954 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
955 List<Coordinate> jp = js.get(jnums).getPoints();
956 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]957 boolean err = false;
[15033]958 for (int nump = 0; nump < ep.size() && !err; ++nump) {
959 Coordinate ept = ep.get(nump);
960 Coordinate jpt = jp.get(nump);
961 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
962 String numtxt = Integer.toString(enums+1);
963 if (enums != jnums) {
964 numtxt += '/' + Integer.toString(jnums+1);
965 }
966 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
967 break;
[14576]968 }
969 }
[15033]970 edone[enums] = true;
971 jdone[jnums] = true;
972 break;
[14576]973 }
974 }
975 }
[15033]976 for (int enums = 0; enums < s.size(); ++enums) {
977 List<Coordinate> ep = s.get(enums).getPoints();
978 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
979 List<Coordinate> jp = js.get(jnums).getPoints();
980 if (!jdone[jnums]) {
981 String numtxt = Integer.toString(enums+1);
982 if (enums != jnums) {
983 numtxt += '/' + Integer.toString(jnums+1);
984 }
[15034]985 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
986 + getDescription(j));
[15033]987 edone[enums] = true;
988 jdone[jnums] = true;
989 break;
[14576]990 }
991 }
992 }
[11410]993 }
994 }
[15034]995 }
996
997 void doMismatchingIcons() throws IOException {
[15033]998 myprintln("*** Mismatching icons: ***");
999 for (String url : eliUrls.keySet()) {
1000 JsonObject e = eliUrls.get(url);
[11420]1001 if (!josmUrls.containsKey(url)) {
[15033]1002 continue;
[11420]1003 }
[15033]1004 ImageryInfo j = josmUrls.get(url);
1005 String ij = getIcon(j);
1006 String ie = getIcon(e);
1007 boolean ijok = isNotBlank(ij);
1008 boolean ieok = isNotBlank(ie);
1009 if (ijok && !ieok) {
1010 if (!optionNoEli) {
1011 myprintln("+ No ELI icon: "+getDescription(j));
[11420]1012 }
[15033]1013 } else if (!ijok && ieok) {
1014 myprintln("- No JOSM icon: "+getDescription(j));
1015 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
[14351]1016 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
1017 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
[14293]1018 ij.startsWith("data:"))) {
[15034]1019 String iehttps = ie.replace("http:", "https:");
[15033]1020 if (ij.equals(iehttps)) {
1021 myprintln("+ Different icons: "+getDescription(j));
[14294]1022 } else {
[15033]1023 myprintln("* Different icons: "+getDescription(j));
[14294]1024 }
[11420]1025 }
1026 }
[15034]1027 }
1028
1029 void doMiscellaneousChecks() throws IOException {
[15033]1030 myprintln("*** Miscellaneous checks: ***");
1031 Map<String, ImageryInfo> josmIds = new HashMap<>();
1032 Collection<String> all = Projections.getAllProjectionCodes();
[13551]1033 DomainValidator dv = DomainValidator.getInstance();
[15033]1034 for (String url : josmUrls.keySet()) {
1035 ImageryInfo j = josmUrls.get(url);
1036 String id = getId(j);
1037 if ("wms".equals(getType(j))) {
1038 String urlLc = url.toLowerCase(Locale.ENGLISH);
1039 if (getProjections(j).isEmpty()) {
1040 myprintln("* WMS without projections: "+getDescription(j));
[13526]1041 } else {
[15033]1042 List<String> unsupported = new LinkedList<>();
1043 List<String> old = new LinkedList<>();
1044 for (String p : getProjectionsUnstripped(j)) {
1045 if ("CRS:84".equals(p)) {
1046 if (!urlLc.contains("version=1.3")) {
1047 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
[13526]1048 }
[15033]1049 } else if (oldproj.containsKey(p)) {
1050 old.add(p);
1051 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1052 unsupported.add(p);
[13526]1053 }
1054 }
[15033]1055 if (!unsupported.isEmpty()) {
1056 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
[13526]1057 }
[15033]1058 for (String o : old) {
[15034]1059 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1060 + getDescription(j));
[13527]1061 }
[13526]1062 }
[15033]1063 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1064 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1065 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1066 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
[13526]1067 }
[13511]1068 }
[15033]1069 List<String> urls = new LinkedList<>();
1070 if (!"scanex".equals(getType(j))) {
1071 urls.add(url);
[13532]1072 }
[15033]1073 String jt = getPermissionReferenceUrl(j);
1074 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1075 urls.add(jt);
1076 jt = getTermsOfUseUrl(j);
1077 if (isNotBlank(jt))
1078 urls.add(jt);
1079 jt = getAttributionUrl(j);
1080 if (isNotBlank(jt))
1081 urls.add(jt);
1082 jt = getIcon(j);
1083 if (isNotBlank(jt) && !jt.startsWith("data:image/png;base64,"))
1084 urls.add(jt);
[15163]1085 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?(/.*)?");
[15033]1086 for (String u : urls) {
[15163]1087 if (!patternU.matcher(u).matches() || u.matches(".*[ \t]+$")) {
[15033]1088 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1089 } else {
[15163]1090 try {
1091 URL jurl = new URL(u.replaceAll("\\{switch:[^\\}]*\\}", "x"));
1092 String domain = jurl.getHost();
1093 int port = jurl.getPort();
1094 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1095 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1096 else if (80 == port || 443 == port) {
1097 myprintln("* Useless port '"+port+"': "+getDescription(j));
1098 }
1099 } catch (MalformedURLException e) {
1100 myprintln("* Malformed URL '"+u+"': "+getDescription(j)+" => "+e.getMessage());
[13554]1101 }
[13551]1102 }
1103 }
1104
[15033]1105 if (josmMirrors.containsKey(url)) {
1106 continue;
[11420]1107 }
[15033]1108 if (isBlank(id)) {
1109 myprintln("* No JOSM-ID: "+getDescription(j));
1110 } else if (josmIds.containsKey(id)) {
1111 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
[11420]1112 } else {
[15033]1113 josmIds.put(id, j);
[11420]1114 }
[15033]1115 String d = getDate(j);
1116 if (isNotBlank(d)) {
1117 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1118 Matcher m = patternD.matcher(d);
1119 if (!m.matches()) {
1120 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
[11572]1121 } else {
1122 try {
[15033]1123 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1124 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1125 if (second.compareTo(first) < 0) {
1126 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
[11572]1127 }
[15034]1128 } catch (Exception e) {
[15033]1129 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
[11572]1130 }
1131 }
[11603]1132 }
[15033]1133 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1134 myprintln("* Attribution link without text: "+getDescription(j));
[12261]1135 }
[15033]1136 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1137 myprintln("* Logo link without image: "+getDescription(j));
[12261]1138 }
[15033]1139 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1140 myprintln("* Terms of Use text without link: "+getDescription(j));
[12261]1141 }
[15033]1142 List<Shape> js = getShapes(j);
1143 if (!js.isEmpty()) {
1144 double minlat = 1000;
1145 double minlon = 1000;
1146 double maxlat = -1000;
1147 double maxlon = -1000;
1148 for (Shape s: js) {
1149 for (Coordinate p: s.getPoints()) {
1150 double lat = p.getLat();
1151 double lon = p.getLon();
[15034]1152 if (lat > maxlat) maxlat = lat;
1153 if (lon > maxlon) maxlon = lon;
1154 if (lat < minlat) minlat = lat;
1155 if (lon < minlon) minlon = lon;
[11422]1156 }
1157 }
[15033]1158 ImageryBounds b = j.getBounds();
1159 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1160 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
[15034]1161 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1162 + getDescription(j));
[11422]1163 }
1164 }
[15033]1165 List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
1166 String cat = getCategory(j);
1167 if (isBlank(cat)) {
1168 myprintln("* No category: "+getDescription(j));
1169 } else if (!knownCategories.contains(cat)) {
1170 myprintln("* Strange category "+cat+": "+getDescription(j));
[13792]1171 }
[11420]1172 }
[7726]1173 }
[9667]1174
[15033]1175 /*
[7726]1176 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1177 */
[15033]1178
[7726]1179 static String getUrl(Object e) {
[15033]1180 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1181 return ((Map<String, JsonObject>) e).get("properties").getString("url");
[7726]1182 }
[15033]1183
[12242]1184 static String getUrlStripped(Object e) {
[15034]1185 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
[12242]1186 }
[15033]1187
[11572]1188 static String getDate(Object e) {
[15033]1189 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1190 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1191 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1192 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
[15034]1193 if (!start.isEmpty() && !end.isEmpty())
[15033]1194 return start+";"+end;
[15034]1195 else if (!start.isEmpty())
[15033]1196 return start+";-";
[15034]1197 else if (!end.isEmpty())
[15033]1198 return "-;"+end;
1199 return "";
[11572]1200 }
[15033]1201
1202 static Date verifyDate(String year, String month, String day) throws ParseException {
1203 String date;
[15034]1204 if (year == null) {
[15033]1205 date = "3000-01-01";
[11854]1206 } else {
[15033]1207 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
[11854]1208 }
[15033]1209 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1210 df.setLenient(false);
1211 return df.parse(date);
[11572]1212 }
[15033]1213
[11420]1214 static String getId(Object e) {
[15033]1215 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1216 return ((Map<String, JsonObject>) e).get("properties").getString("id");
[11420]1217 }
[15033]1218
[7726]1219 static String getName(Object e) {
[15033]1220 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1221 return ((Map<String, JsonObject>) e).get("properties").getString("name");
[7726]1222 }
[15033]1223
1224 static List<ImageryInfo> getMirrors(Object e) {
1225 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1226 return Collections.emptyList();
[11967]1227 }
[15033]1228
1229 static List<String> getProjections(Object e) {
1230 List<String> r = new ArrayList<>();
1231 List<String> u = getProjectionsUnstripped(e);
1232 if (u != null) {
1233 for (String p : u) {
1234 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1235 r.add(p);
[13530]1236 }
1237 }
1238 }
[15033]1239 return r;
[13530]1240 }
[15033]1241
1242 static List<String> getProjectionsUnstripped(Object e) {
1243 List<String> r = null;
[11975]1244 if (e instanceof ImageryInfo) {
[15033]1245 r = ((ImageryInfo) e).getServerProjections();
[11975]1246 } else {
[15033]1247 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1248 if (s != null) {
1249 r = new ArrayList<>();
1250 for (JsonValue p : s.asJsonArray()) {
1251 r.add(((JsonString) p).getString());
[13530]1252 }
[11981]1253 }
[11975]1254 }
[15033]1255 return r != null ? r : Collections.emptyList();
[11975]1256 }
[15033]1257
[11410]1258 static List<Shape> getShapes(Object e) {
1259 if (e instanceof ImageryInfo) {
[15033]1260 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
[15034]1261 if (bounds != null) {
[15033]1262 return bounds.getShapes();
[11411]1263 }
[15033]1264 return Collections.emptyList();
[11410]1265 }
[15033]1266 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1267 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1268 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1269 List<Shape> l = new ArrayList<>();
1270 for (JsonValue shapes: poly) {
1271 Shape s = new Shape();
1272 for (JsonValue point: shapes.asJsonArray()) {
1273 String lon = point.asJsonArray().getJsonNumber(0).toString();
1274 String lat = point.asJsonArray().getJsonNumber(1).toString();
1275 s.addPoint(lat, lon);
[11410]1276 }
[15033]1277 l.add(s);
[11410]1278 }
[15033]1279 return l;
[11410]1280 }
[15033]1281 return Collections.emptyList();
[11410]1282 }
[15033]1283
[7726]1284 static String getType(Object e) {
[15033]1285 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1286 return ((Map<String, JsonObject>) e).get("properties").getString("type");
[7726]1287 }
[15033]1288
[7726]1289 static Integer getMinZoom(Object e) {
1290 if (e instanceof ImageryInfo) {
[15033]1291 int mz = ((ImageryInfo) e).getMinZoom();
1292 return mz == 0 ? null : mz;
[7726]1293 } else {
[15033]1294 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1295 if (num == null) return null;
1296 return num.intValue();
[7726]1297 }
1298 }
[15033]1299
[7726]1300 static Integer getMaxZoom(Object e) {
1301 if (e instanceof ImageryInfo) {
[15033]1302 int mz = ((ImageryInfo) e).getMaxZoom();
1303 return mz == 0 ? null : mz;
[7726]1304 } else {
[15033]1305 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1306 if (num == null) return null;
1307 return num.intValue();
[7726]1308 }
1309 }
[15033]1310
[7726]1311 static String getCountryCode(Object e) {
[15033]1312 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1313 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
[7726]1314 }
[15033]1315
[9505]1316 static String getQuality(Object e) {
[15033]1317 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1318 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1319 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
[9505]1320 }
[15033]1321
1322 static boolean getOverlay(Object e) {
1323 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1324 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1325 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
[13536]1326 }
[15033]1327
[11420]1328 static String getIcon(Object e) {
[15033]1329 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1330 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
[11420]1331 }
[15033]1332
[11975]1333 static String getAttributionText(Object e) {
[15033]1334 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1335 try {
1336 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1337 } catch (NullPointerException ex) {
1338 return null;
1339 }
[11975]1340 }
[15033]1341
[11975]1342 static String getAttributionUrl(Object e) {
[15033]1343 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1344 try {
1345 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1346 } catch (NullPointerException ex) {
1347 return null;
1348 }
[11975]1349 }
[15033]1350
[11975]1351 static String getTermsOfUseText(Object e) {
[15033]1352 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1353 return null;
[11975]1354 }
[15033]1355
[11975]1356 static String getTermsOfUseUrl(Object e) {
[15033]1357 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1358 return null;
[11975]1359 }
[15033]1360
[13792]1361 static String getCategory(Object e) {
1362 if (e instanceof ImageryInfo) {
[15033]1363 return ((ImageryInfo) e).getImageryCategoryOriginalString();
[13792]1364 }
[15033]1365 return null;
[13792]1366 }
[15033]1367
[12261]1368 static String getLogoImage(Object e) {
[15033]1369 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1370 return null;
[12261]1371 }
[15033]1372
[12261]1373 static String getLogoUrl(Object e) {
[15033]1374 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1375 return null;
[12261]1376 }
[15033]1377
[11975]1378 static String getPermissionReferenceUrl(Object e) {
[15033]1379 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1380 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
[11975]1381 }
[15033]1382
[15082]1383 static Map<String, Set<String>> getNoTileHeader(Object e) {
1384 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getNoTileHeaders();
1385 JsonObject nth = ((Map<String, JsonObject>) e).get("properties").getJsonObject("no_tile_header");
1386 return nth == null ? null : nth.keySet().stream().collect(Collectors.toMap(
1387 Function.identity(),
1388 k -> nth.getJsonArray(k).stream().map(x -> ((JsonString) x).getString()).collect(Collectors.toSet())));
1389 }
1390
[15034]1391 static Map<String, String> getDescriptions(Object e) {
[15082]1392 Map<String, String> res = new HashMap<>();
[11975]1393 if (e instanceof ImageryInfo) {
[15033]1394 String a = ((ImageryInfo) e).getDescription();
1395 if (a != null) res.put("en", a);
[11975]1396 } else {
[15033]1397 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
[15034]1398 if (a != null) res.put("en", a.replaceAll("''", "'"));
[11975]1399 }
[15033]1400 return res;
[11975]1401 }
[15033]1402
1403 static boolean getValidGeoreference(Object e) {
1404 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1405 return false;
[11975]1406 }
[15033]1407
1408 static boolean getDefault(Object e) {
1409 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1410 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
[12226]1411 }
[15033]1412
[7726]1413 String getDescription(Object o) {
[15033]1414 String url = getUrl(o);
1415 String cc = getCountryCode(o);
[7726]1416 if (cc == null) {
[15033]1417 ImageryInfo j = josmUrls.get(url);
1418 if (j != null) cc = getCountryCode(j);
[7726]1419 if (cc == null) {
[15033]1420 JsonObject e = eliUrls.get(url);
1421 if (e != null) cc = getCountryCode(e);
[7726]1422 }
1423 }
1424 if (cc == null) {
[15033]1425 cc = "";
[7726]1426 } else {
[15033]1427 cc = "["+cc+"] ";
[7726]1428 }
[15033]1429 String d = cc + getName(o) + " - " + getUrl(o);
1430 if (optionShorten) {
1431 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
[7726]1432 }
[15033]1433 return d;
[7726]1434 }
1435}
Note: See TracBrowser for help on using the repository browser.