Context Navigation

source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15053

Last change on this file since 15053 was 15037, checked in by stoecker, 7 years ago
suppress unchecked warnings
Property svn:eol-style set to `native`
File size: 60.9 KB

Rev	Line
[7726]	1	// License: GPL. For details, see LICENSE file.
[15034]	2	import static java.nio.charset.StandardCharsets.UTF_8;
[15033]	3	import static org.apache.commons.lang3.StringUtils.isBlank;
	4	import static org.apache.commons.lang3.StringUtils.isNotBlank;
	5
	6	import java.io.BufferedReader;
	7	import java.io.IOException;
	8	import java.io.InputStreamReader;
[15034]	9	import java.io.OutputStream;
[15033]	10	import java.io.OutputStreamWriter;
	11	import java.lang.reflect.Field;
[15034]	12	import java.nio.file.Files;
	13	import java.nio.file.Paths;
[15033]	14	import java.text.DecimalFormat;
	15	import java.text.ParseException;
	16	import java.text.SimpleDateFormat;
	17	import java.util.ArrayList;
	18	import java.util.Arrays;
	19	import java.util.Calendar;
	20	import java.util.Collection;
	21	import java.util.Collections;
	22	import java.util.Date;
	23	import java.util.HashMap;
	24	import java.util.LinkedList;
	25	import java.util.List;
	26	import java.util.Locale;
	27	import java.util.Map;
	28	import java.util.Map.Entry;
	29	import java.util.Objects;
	30	import java.util.regex.Matcher;
	31	import java.util.regex.Pattern;
	32	import java.util.stream.Collectors;
	33
	34	import javax.json.Json;
	35	import javax.json.JsonArray;
	36	import javax.json.JsonNumber;
	37	import javax.json.JsonObject;
	38	import javax.json.JsonReader;
	39	import javax.json.JsonString;
	40	import javax.json.JsonValue;
	41
	42	import org.openstreetmap.gui.jmapviewer.Coordinate;
	43	import org.openstreetmap.josm.data.Preferences;
	44	import org.openstreetmap.josm.data.imagery.ImageryInfo;
	45	import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
	46	import org.openstreetmap.josm.data.imagery.Shape;
	47	import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
	48	import org.openstreetmap.josm.data.projection.Projections;
	49	import org.openstreetmap.josm.data.validation.routines.DomainValidator;
	50	import org.openstreetmap.josm.io.imagery.ImageryReader;
	51	import org.openstreetmap.josm.spi.preferences.Config;
[15034]	52	import org.openstreetmap.josm.tools.Logging;
[15033]	53	import org.openstreetmap.josm.tools.OptionParser;
	54	import org.openstreetmap.josm.tools.OptionParser.OptionCount;
	55	import org.openstreetmap.josm.tools.ReflectionUtils;
	56	import org.xml.sax.SAXException;
	57
[7726]	58	/**
[11854]	59	* Compare and analyse the differences of the editor layer index and the JOSM imagery list.
[7726]	60	* The goal is to keep both lists in sync.
	61	*
[11854]	62	* The editor layer index project (https://github.com/osmlab/editor-layer-index)
[11694]	63	* provides also a version in the JOSM format, but the GEOJSON is the original source
[7726]	64	* format, so we read that.
	65	*
	66	* How to run:
	67	* -----------
	68	*
	69	* Main JOSM binary needs to be in classpath, e.g.
	70	*
[15033]	71	* $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
[9667]	72	*
[7726]	73	* Add option "-h" to show the available command line flags.
	74	*/
[15037]	75	@SuppressWarnings("unchecked")
[15033]	76	public class SyncEditorLayerIndex {
[14019]	77
[15034]	78	private static final int MAXLEN = 140;
	79
[15033]	80	private List<ImageryInfo> josmEntries;
	81	private JsonArray eliEntries;
[7726]	82
[15034]	83	private final Map<String, JsonObject> eliUrls = new HashMap<>();
	84	private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
	85	private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
	86	private static final Map<String, String> oldproj = new HashMap<>();
	87	private static final List<String> ignoreproj = new LinkedList<>();
[7726]	88
[15033]	89	private static String eliInputFile = "imagery_eli.geojson";
	90	private static String josmInputFile = "imagery_josm.imagery.xml";
	91	private static String ignoreInputFile = "imagery_josm.ignores.txt";
[15034]	92	private static OutputStream outputFile;
	93	private static OutputStreamWriter outputStream;
[15033]	94	private static String optionOutput;
	95	private static boolean optionShorten;
	96	private static boolean optionNoSkip;
	97	private static boolean optionXhtmlBody;
	98	private static boolean optionXhtml;
	99	private static String optionEliXml;
	100	private static String optionJosmXml;
	101	private static String optionEncoding;
	102	private static boolean optionNoEli;
	103	private Map<String, String> skip = new HashMap<>();
[7726]	104
	105	/**
	106	* Main method.
[15033]	107	* @param args program arguments
	108	* @throws IOException if any I/O error occurs
	109	* @throws ReflectiveOperationException if any reflective operation error occurs
	110	* @throws SAXException if any SAX error occurs
[7726]	111	*/
[15033]	112	public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
	113	Locale.setDefault(Locale.ROOT);
[15034]	114	parseCommandLineArguments(args);
[15033]	115	Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
	116	Config.setPreferencesInstance(pref);
	117	pref.init(false);
	118	SyncEditorLayerIndex script = new SyncEditorLayerIndex();
	119	script.setupProj();
	120	script.loadSkip();
	121	script.start();
	122	script.loadJosmEntries();
	123	if (optionJosmXml != null) {
[15034]	124	try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
	125	script.printentries(script.josmEntries, stream);
	126	}
[11964]	127	}
[15033]	128	script.loadELIEntries();
	129	if (optionEliXml != null) {
[15034]	130	try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
	131	script.printentries(script.eliEntries, stream);
	132	}
[11964]	133	}
[15033]	134	script.checkInOneButNotTheOther();
	135	script.checkCommonEntries();
	136	script.end();
	137	if (outputStream != null) {
	138	outputStream.close();
[9505]	139	}
[15033]	140	if (outputFile != null) {
	141	outputFile.close();
[9505]	142	}
[7726]	143	}
[9653]	144
[7726]	145	/**
[15033]	146	* Displays help on the console
	147	*/
	148	private static void showHelp() {
	149	System.out.println(getHelp());
	150	System.exit(0);
	151	}
	152
	153	static String getHelp() {
	154	return "usage: java -cp build SyncEditorLayerIndex\n" +
	155	"-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
[15034]	156	"-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
	157	"Default is imagery_eli.geojson (current directory).\n" +
[15033]	158	"-h,--help show this help\n" +
	159	"-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
[15034]	160	"-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
	161	"Default is imagery_josm.imagery.xml (current directory).\n" +
[15033]	162	"-m,--noeli don't show output for ELI problems\n" +
	163	"-n,--noskip don't skip known entries\n" +
	164	"-o,--output <output> Output file, - prints to stdout (default: -)\n" +
	165	"-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
	166	"-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
	167	"-s,--shorten shorten the output, so it is easier to read in a console window\n" +
	168	"-x,--xhtmlbody create XHTML body for display in a web page\n" +
	169	"-X,--xhtml create XHTML for display in a web page\n";
	170	}
	171
	172	/**
[7726]	173	* Parse command line arguments.
[15033]	174	* @param args program arguments
[15034]	175	* @throws IOException in case of I/O error
[7726]	176	*/
[15034]	177	static void parseCommandLineArguments(String[] args) throws IOException {
[15033]	178	new OptionParser("JOSM/ELI synchronization script")
	179	.addFlagParameter("help", SyncEditorLayerIndex::showHelp)
	180	.addShortAlias("help", "h")
	181	.addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
	182	.addShortAlias("output", "o")
	183	.addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
	184	.addShortAlias("eli_input", "e")
	185	.addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
	186	.addShortAlias("josm_input", "j")
	187	.addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
	188	.addShortAlias("ignore_input", "i")
	189	.addFlagParameter("shorten", () -> optionShorten = true)
	190	.addShortAlias("shorten", "s")
	191	.addFlagParameter("noskip", () -> optionNoSkip = true)
	192	.addShortAlias("noskip", "n")
	193	.addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
	194	.addShortAlias("xhtmlbody", "x")
	195	.addFlagParameter("xhtml", () -> optionXhtml = true)
	196	.addShortAlias("xhtml", "X")
	197	.addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
	198	.addShortAlias("elixml", "p")
	199	.addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
	200	.addShortAlias("josmxml", "q")
	201	.addFlagParameter("noeli", () -> optionNoEli = true)
	202	.addShortAlias("noeli", "m")
	203	.addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
	204	.addShortAlias("encoding", "c")
	205	.parseOptionsOrExit(Arrays.asList(args));
[7726]	206
[15034]	207	if (optionOutput != null && !"-".equals(optionOutput)) {
	208	outputFile = Files.newOutputStream(Paths.get(optionOutput));
[15033]	209	outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
	210	} else if (optionEncoding != null) {
	211	outputStream = new OutputStreamWriter(System.out, optionEncoding);
[7726]	212	}
	213	}
	214
[13530]	215	void setupProj() {
[15033]	216	oldproj.put("EPSG:3359", "EPSG:3404");
	217	oldproj.put("EPSG:3785", "EPSG:3857");
	218	oldproj.put("EPSG:31297", "EPGS:31287");
	219	oldproj.put("EPSG:31464", "EPSG:31468");
	220	oldproj.put("EPSG:54004", "EPSG:3857");
	221	oldproj.put("EPSG:102100", "EPSG:3857");
	222	oldproj.put("EPSG:102113", "EPSG:3857");
	223	oldproj.put("EPSG:900913", "EPGS:3857");
	224	ignoreproj.add("EPSG:4267");
	225	ignoreproj.add("EPSG:5221");
	226	ignoreproj.add("EPSG:5514");
	227	ignoreproj.add("EPSG:32019");
	228	ignoreproj.add("EPSG:102066");
	229	ignoreproj.add("EPSG:102067");
	230	ignoreproj.add("EPSG:102685");
	231	ignoreproj.add("EPSG:102711");
[13530]	232	}
	233
[15033]	234	void loadSkip() throws IOException {
	235	final Pattern pattern = Pattern.compile("^\\\|\\\| (ELI\|Ignore) \\\|\\\| \\{\\{\\{(.+)\\}\\}\\} \\\|\\\|");
[15034]	236	try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
[15033]	237	String line;
[11238]	238
[15033]	239	while ((line = fr.readLine()) != null) {
	240	Matcher res = pattern.matcher(line);
	241	if (res.matches()) {
	242	if ("Ignore".equals(res.group(1))) {
	243	skip.put(res.group(2), "green");
	244	} else {
	245	skip.put(res.group(2), "darkgoldenrod");
	246	}
[11238]	247	}
	248	}
[11234]	249	}
[11238]	250	}
[9653]	251
[15033]	252	void myprintlnfinal(String s) throws IOException {
	253	if (outputStream != null) {
	254	outputStream.write(s + System.getProperty("line.separator"));
[9658]	255	} else {
[15033]	256	System.out.println(s);
[9658]	257	}
	258	}
	259
[15033]	260	void myprintln(String s) throws IOException {
	261	if (skip.containsKey(s)) {
	262	String color = skip.get(s);
	263	skip.remove(s);
	264	if (optionXhtmlBody \|\| optionXhtml) {
[15034]	265	s = "<pre style=\"margin:3px;color:"+color+"\">"
	266	+ s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">")+"</pre>";
[9658]	267	}
[15033]	268	if (!optionNoSkip) {
	269	return;
[9662]	270	}
[15034]	271	} else if (optionXhtmlBody \|\| optionXhtml) {
[15033]	272	String color =
	273	s.startsWith("***") ? "black" :
	274	((s.startsWith("+ ") \|\| s.startsWith("+++ ELI")) ? "blue" :
	275	(s.startsWith("#") ? "indigo" :
	276	(s.startsWith("!") ? "orange" : "red")));
[15034]	277	s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">")+"</pre>";
[9505]	278	}
[15033]	279	if ((s.startsWith("+ ") \|\| s.startsWith("+++ ELI") \|\| s.startsWith("#")) && optionNoEli) {
	280	return;
[11965]	281	}
[15033]	282	myprintlnfinal(s);
[9658]	283	}
	284
[15033]	285	void start() throws IOException {
	286	if (optionXhtml) {
[15034]	287	myprintlnfinal(
	288	"<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
	289	myprintlnfinal(
	290	"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
	291	"<title>JOSM - ELI differences</title></head><body>\n");
[9505]	292	}
	293	}
[9653]	294
[15033]	295	void end() throws IOException {
	296	for (String s : skip.keySet()) {
	297	myprintln("+++ Obsolete skip entry: " + s);
[9658]	298	}
[15033]	299	if (optionXhtml) {
	300	myprintlnfinal("</body></html>\n");
[9658]	301	}
	302	}
	303
[15033]	304	void loadELIEntries() throws IOException {
[15034]	305	try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
[15033]	306	eliEntries = jr.readObject().getJsonArray("features");
	307	}
[9653]	308
[15033]	309	for (JsonValue e : eliEntries) {
	310	String url = getUrlStripped(e);
[9653]	311	if (url.contains("{z}")) {
[15033]	312	myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
[15034]	313	url = url.replace("{z}", "{zoom}");
[9653]	314	}
[11582]	315	if (eliUrls.containsKey(url)) {
[15033]	316	myprintln("+++ ELI-URL is not unique: "+url);
[9505]	317	} else {
[15033]	318	eliUrls.put(url, e.asJsonObject());
[9505]	319	}
[15033]	320	JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
	321	if (s != null) {
	322	String urlLc = url.toLowerCase(Locale.ENGLISH);
	323	List<String> old = new LinkedList<>();
	324	for (JsonValue p : s) {
	325	String proj = ((JsonString) p).getString();
	326	if (oldproj.containsKey(proj) \|\| ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
	327	old.add(proj);
[13530]	328	}
	329	}
[15033]	330	if (!old.isEmpty()) {
	331	myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
[13530]	332	}
	333	}
[7726]	334	}
[15033]	335	myprintln("* Loaded "+eliEntries.size()+" entries (ELI). *");
[7726]	336	}
[15033]	337
	338	String cdata(String s) {
	339	return cdata(s, false);
[11968]	340	}
[7726]	341
[15033]	342	String cdata(String s, boolean escape) {
	343	if (escape) {
	344	return s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
	345	} else if (s.matches("[<>&]"))
	346	return "<![CDATA["+s+"]]>";
	347	return s;
	348	}
	349
	350	String maininfo(Object entry, String offset) {
	351	String t = getType(entry);
	352	String res = offset + "<type>"+t+"</type>\n";
	353	res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
	354	if (getMinZoom(entry) != null)
	355	res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
	356	if (getMaxZoom(entry) != null)
	357	res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
	358	if ("wms".equals(t)) {
	359	List<String> p = getProjections(entry);
	360	if (p != null) {
	361	res += offset + "<projections>\n";
[15034]	362	for (String c : p) {
[15033]	363	res += offset + " <code>"+c+"</code>\n";
[15034]	364	}
[15033]	365	res += offset + "</projections>\n";
[11975]	366	}
[11967]	367	}
[15033]	368	return res;
[11967]	369	}
[12061]	370
[15033]	371	void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
	372	DecimalFormat df = new DecimalFormat("#.#######");
	373	df.setRoundingMode(java.math.RoundingMode.CEILING);
	374	stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
	375	stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
	376	for (Object e : entries) {
[13536]	377	stream.write(" <entry"
[15034]	378	+ ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
	379	+ (getOverlay(e) ? " overlay=\"true\"" : "")
[15033]	380	+ ">\n");
	381	String t;
	382	if (isNotBlank(t = getName(e)))
	383	stream.write(" <name>"+cdata(t, true)+"</name>\n");
	384	if (isNotBlank(t = getId(e)))
	385	stream.write(" <id>"+t+"</id>\n");
	386	if (isNotBlank(t = getCategory(e)))
	387	stream.write(" <category>"+t+"</category>\n");
	388	if (isNotBlank(t = getDate(e)))
	389	stream.write(" <date>"+t+"</date>\n");
	390	if (isNotBlank(t = getCountryCode(e)))
	391	stream.write(" <country-code>"+t+"</country-code>\n");
	392	if ((getDefault(e)))
	393	stream.write(" <default>true</default>\n");
	394	stream.write(maininfo(e, " "));
	395	if (isNotBlank(t = getAttributionText(e)))
	396	stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
	397	if (isNotBlank(t = getAttributionUrl(e)))
	398	stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
	399	if (isNotBlank(t = getLogoImage(e)))
	400	stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
	401	if (isNotBlank(t = getLogoUrl(e)))
	402	stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
	403	if (isNotBlank(t = getTermsOfUseText(e)))
	404	stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
	405	if (isNotBlank(t = getTermsOfUseUrl(e)))
	406	stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
	407	if (isNotBlank(t = getPermissionReferenceUrl(e)))
	408	stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
	409	if ((getValidGeoreference(e)))
	410	stream.write(" <valid-georeference>true</valid-georeference>\n");
	411	if (isNotBlank(t = getIcon(e)))
	412	stream.write(" <icon>"+cdata(t)+"</icon>\n");
	413	for (Entry<String, String> d : getDescriptions(e).entrySet()) {
	414	stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
[11975]	415	}
[15033]	416	for (ImageryInfo m : getMirrors(e)) {
	417	stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
[11967]	418	}
[15033]	419	double minlat = 1000;
	420	double minlon = 1000;
	421	double maxlat = -1000;
	422	double maxlon = -1000;
	423	String shapes = "";
	424	String sep = "\n ";
[13771]	425	try {
[15033]	426	for (Shape s: getShapes(e)) {
	427	shapes += " <shape>";
	428	int i = 0;
	429	for (Coordinate p: s.getPoints()) {
	430	double lat = p.getLat();
	431	double lon = p.getLon();
	432	if (lat > maxlat) maxlat = lat;
	433	if (lon > maxlon) maxlon = lon;
	434	if (lat < minlat) minlat = lat;
	435	if (lon < minlon) minlon = lon;
[15034]	436	if ((i++ % 3) == 0) {
[15033]	437	shapes += sep + " ";
[13771]	438	}
[15033]	439	shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
[11964]	440	}
[15033]	441	shapes += sep + "</shape>\n";
[11964]	442	}
[15034]	443	} catch (IllegalArgumentException ignored) {
	444	Logging.trace(ignored);
[11964]	445	}
[15033]	446	if (!shapes.isEmpty()) {
[15034]	447	stream.write(" <bounds min-lat='"+df.format(minlat)
	448	+"' min-lon='"+df.format(minlon)
	449	+"' max-lat='"+df.format(maxlat)
	450	+"' max-lon='"+df.format(maxlon)+"'>\n");
[15033]	451	stream.write(shapes + " </bounds>\n");
[11964]	452	}
[15033]	453	stream.write(" </entry>\n");
[11964]	454	}
[15033]	455	stream.write("</imagery>\n");
	456	stream.close();
[11964]	457	}
	458
[15033]	459	void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
	460	try (ImageryReader reader = new ImageryReader(josmInputFile)) {
	461	josmEntries = reader.parse();
	462	}
[9667]	463
[15033]	464	for (ImageryInfo e : josmEntries) {
	465	if (isBlank(getUrl(e))) {
	466	myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
	467	continue;
[14554]	468	}
[15033]	469	if (isBlank(getName(e))) {
	470	myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
	471	continue;
[14554]	472	}
[15033]	473	String url = getUrlStripped(e);
[9658]	474	if (url.contains("{z}")) {
[15033]	475	myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
[15034]	476	url = url.replace("{z}", "{zoom}");
[9658]	477	}
[9505]	478	if (josmUrls.containsKey(url)) {
[15033]	479	myprintln("+++ JOSM-URL is not unique: "+url);
[9505]	480	} else {
[15033]	481	josmUrls.put(url, e);
[7726]	482	}
[15033]	483	for (ImageryInfo m : e.getMirrors()) {
	484	url = getUrlStripped(m);
	485	Field origNameField = ImageryInfo.class.getDeclaredField("origName");
	486	ReflectionUtils.setObjectsAccessible(origNameField);
[15034]	487	origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
[9658]	488	if (josmUrls.containsKey(url)) {
[15033]	489	myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
[9658]	490	} else {
[15033]	491	josmUrls.put(url, m);
	492	josmMirrors.put(url, m);
[9658]	493	}
	494	}
[7726]	495	}
[15033]	496	myprintln("* Loaded "+josmEntries.size()+" entries (JOSM). *");
[7726]	497	}
	498
[15033]	499	void checkInOneButNotTheOther() throws IOException {
	500	List<String> le = new LinkedList<>(eliUrls.keySet());
	501	List<String> lj = new LinkedList<>(josmUrls.keySet());
[13593]	502
[15033]	503	List<String> ke = new LinkedList<>(le);
	504	for (String url : ke) {
	505	if (lj.contains(url)) {
	506	le.remove(url);
	507	lj.remove(url);
[13593]	508	}
	509	}
	510
[15033]	511	if (!le.isEmpty() && !lj.isEmpty()) {
	512	ke = new LinkedList<>(le);
	513	for (String urle : ke) {
	514	JsonObject e = eliUrls.get(urle);
	515	String ide = getId(e);
[15034]	516	String urlhttps = urle.replace("http:", "https:");
[15033]	517	if (lj.contains(urlhttps)) {
	518	myprintln("+ Missing https: "+getDescription(e));
	519	eliUrls.put(urlhttps, eliUrls.get(urle));
	520	eliUrls.remove(urle);
	521	le.remove(urle);
	522	lj.remove(urlhttps);
	523	} else if (isNotBlank(ide)) {
	524	List<String> kj = new LinkedList<>(lj);
	525	for (String urlj : kj) {
	526	ImageryInfo j = josmUrls.get(urlj);
	527	String idj = getId(j);
[13714]	528
[15033]	529	if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
	530	myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
	531	le.remove(urle);
	532	lj.remove(urlj);
	533	// replace key for this entry with JOSM URL
	534	eliUrls.remove(urle);
[15034]	535	eliUrls.put(urlj, e);
[15033]	536	break;
[13593]	537	}
	538	}
[13517]	539	}
[7726]	540	}
	541	}
[13714]	542
[15033]	543	myprintln("* URLs found in ELI but not in JOSM ("+le.size()+"): *");
	544	Collections.sort(le);
[13593]	545	if (!le.isEmpty()) {
[15033]	546	for (String l : le) {
	547	myprintln("- " + getDescription(eliUrls.get(l)));
[11412]	548	}
[7726]	549	}
[15033]	550	myprintln("* URLs found in JOSM but not in ELI ("+lj.size()+"): *");
	551	Collections.sort(lj);
[13593]	552	if (!lj.isEmpty()) {
[15033]	553	for (String l : lj) {
	554	myprintln("+ " + getDescription(josmUrls.get(l)));
[11412]	555	}
[7726]	556	}
	557	}
[9667]	558
[15033]	559	void checkCommonEntries() throws IOException {
[15034]	560	doSameUrlButDifferentName();
	561	doSameUrlButDifferentId();
	562	doSameUrlButDifferentType();
	563	doSameUrlButDifferentZoomBounds();
	564	doSameUrlButDifferentCountryCode();
	565	doSameUrlButDifferentQuality();
	566	doSameUrlButDifferentDates();
	567	doSameUrlButDifferentInformation();
	568	doMismatchingShapes();
	569	doMismatchingIcons();
	570	doMiscellaneousChecks();
	571	}
	572
	573	void doSameUrlButDifferentName() throws IOException {
[15033]	574	myprintln("* Same URL, but different name: *");
	575	for (String url : eliUrls.keySet()) {
	576	JsonObject e = eliUrls.get(url);
	577	if (!josmUrls.containsKey(url)) continue;
	578	ImageryInfo j = josmUrls.get(url);
[15034]	579	String ename = getName(e).replace("'", "\u2019");
	580	String jname = getName(j).replace("'", "\u2019");
[11951]	581	if (!ename.equals(jname)) {
[15033]	582	myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
[7726]	583	}
	584	}
[15034]	585	}
[9667]	586
[15034]	587	void doSameUrlButDifferentId() throws IOException {
[15033]	588	myprintln("* Same URL, but different Id: *");
	589	for (String url : eliUrls.keySet()) {
	590	JsonObject e = eliUrls.get(url);
	591	if (!josmUrls.containsKey(url)) continue;
	592	ImageryInfo j = josmUrls.get(url);
	593	String ename = getId(e);
	594	String jname = getId(j);
	595	if (!Objects.equals(ename, jname)) {
	596	myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
[12126]	597	}
[12226]	598	}
[15034]	599	}
[12126]	600
[15034]	601	void doSameUrlButDifferentType() throws IOException {
[15033]	602	myprintln("* Same URL, but different type: *");
	603	for (String url : eliUrls.keySet()) {
	604	JsonObject e = eliUrls.get(url);
	605	if (!josmUrls.containsKey(url)) continue;
	606	ImageryInfo j = josmUrls.get(url);
	607	if (!Objects.equals(getType(e), getType(j))) {
	608	myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
[7726]	609	}
	610	}
[15034]	611	}
[9667]	612
[15034]	613	void doSameUrlButDifferentZoomBounds() throws IOException {
[15033]	614	myprintln("* Same URL, but different zoom bounds: *");
	615	for (String url : eliUrls.keySet()) {
	616	JsonObject e = eliUrls.get(url);
	617	if (!josmUrls.containsKey(url)) continue;
	618	ImageryInfo j = josmUrls.get(url);
[7726]	619
[15033]	620	Integer eMinZoom = getMinZoom(e);
	621	Integer jMinZoom = getMinZoom(j);
[13997]	622	/* dont warn for entries copied from the base of the mirror */
[15033]	623	if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]	624	jMinZoom = null;
[15033]	625	if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
	626	myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
[7726]	627	}
[15033]	628	Integer eMaxZoom = getMaxZoom(e);
	629	Integer jMaxZoom = getMaxZoom(j);
[13997]	630	/* dont warn for entries copied from the base of the mirror */
[15033]	631	if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]	632	jMaxZoom = null;
[15033]	633	if (!Objects.equals(eMaxZoom, jMaxZoom)) {
	634	myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
[7726]	635	}
	636	}
[15034]	637	}
[9667]	638
[15034]	639	void doSameUrlButDifferentCountryCode() throws IOException {
[15033]	640	myprintln("* Same URL, but different country code: *");
	641	for (String url : eliUrls.keySet()) {
	642	JsonObject e = eliUrls.get(url);
	643	if (!josmUrls.containsKey(url)) continue;
	644	ImageryInfo j = josmUrls.get(url);
	645	String cce = getCountryCode(e);
[13931]	646	if ("ZZ".equals(cce)) { /* special ELI country code */
[15033]	647	cce = null;
[13931]	648	}
[15033]	649	if (cce != null && !cce.equals(getCountryCode(j))) {
	650	myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
[7726]	651	}
	652	}
[15034]	653	}
	654
	655	void doSameUrlButDifferentQuality() throws IOException {
[15033]	656	myprintln("* Same URL, but different quality: *");
	657	for (String url : eliUrls.keySet()) {
	658	JsonObject e = eliUrls.get(url);
[9515]	659	if (!josmUrls.containsKey(url)) {
[15033]	660	String q = getQuality(e);
	661	if ("eli-best".equals(q)) {
	662	myprintln("- Quality best entry not in JOSM for "+getDescription(e));
[9515]	663	}
[15033]	664	continue;
[9515]	665	}
[15033]	666	ImageryInfo j = josmUrls.get(url);
	667	if (!Objects.equals(getQuality(e), getQuality(j))) {
	668	myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
[9505]	669	}
[11599]	670	}
[15034]	671	}
	672
	673	void doSameUrlButDifferentDates() throws IOException {
[15033]	674	myprintln("* Same URL, but different dates: *");
	675	Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
	676	for (String url : eliUrls.keySet()) {
	677	String ed = getDate(eliUrls.get(url));
	678	if (!josmUrls.containsKey(url)) continue;
	679	ImageryInfo j = josmUrls.get(url);
	680	String jd = getDate(j);
[11612]	681	// The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
[15034]	682	String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]	683	// ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
[15033]	684	String ed2 = ed;
	685	Matcher m = pattern.matcher(ed);
	686	if (m.matches()) {
	687	Calendar cal = Calendar.getInstance();
	688	cal.set(Integer.valueOf(m.group(2)),
	689	m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
	690	m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
	691	cal.add(Calendar.DAY_OF_MONTH, -1);
	692	ed2 = m.group(1) + cal.get(Calendar.YEAR);
	693	if (m.group(4) != null)
	694	ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
	695	if (m.group(6) != null)
	696	ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
[11639]	697	}
[15034]	698	String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]	699	if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
[15033]	700	String t = "'"+ed+"'";
[11612]	701	if (!ed.equals(ef)) {
[15033]	702	t += " or '"+ef+"'";
[11612]	703	}
[11666]	704	if (jd.isEmpty()) {
[15033]	705	myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
[11668]	706	} else if (!ed.isEmpty()) {
[15033]	707	myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
	708	} else if (!optionNoEli) {
	709	myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
[11666]	710	}
[11573]	711	}
[11665]	712	}
[15034]	713	}
	714
	715	void doSameUrlButDifferentInformation() throws IOException {
[15033]	716	myprintln("* Same URL, but different information: *");
	717	for (String url : eliUrls.keySet()) {
	718	if (!josmUrls.containsKey(url)) continue;
	719	JsonObject e = eliUrls.get(url);
	720	ImageryInfo j = josmUrls.get(url);
[11981]	721
[15033]	722	String et = getDescriptions(e).getOrDefault("en", "");
	723	String jt = getDescriptions(j).getOrDefault("en", "");
[12086]	724	if (!et.equals(jt)) {
[15033]	725	if (jt.isEmpty()) {
	726	myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
	727	} else if (!et.isEmpty()) {
	728	myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
	729	} else if (!optionNoEli) {
	730	myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
[11981]	731	}
	732	}
	733
[15033]	734	et = getPermissionReferenceUrl(e);
	735	jt = getPermissionReferenceUrl(j);
	736	String jt2 = getTermsOfUseUrl(j);
	737	if (isBlank(jt)) jt = jt2;
	738	if (!Objects.equals(et, jt)) {
	739	if (isBlank(jt)) {
	740	myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
	741	} else if (isNotBlank(et)) {
	742	String ethttps = et.replace("http:", "https:");
	743	if (isBlank(jt2) \|\| !(jt2.equals(ethttps) \|\| jt2.equals(et+"/") \|\| jt2.equals(ethttps+"/"))) {
	744	if (jt.equals(ethttps) \|\| jt.equals(et+"/") \|\| jt.equals(ethttps+"/")) {
	745	myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13578]	746	} else {
[15033]	747	String ja = getAttributionUrl(j);
	748	if (ja != null && (ja.equals(et) \|\| ja.equals(ethttps) \|\| ja.equals(et+"/") \|\| ja.equals(ethttps+"/"))) {
	749	myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
[13579]	750	} else {
[15033]	751	myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13579]	752	}
[13578]	753	}
[13551]	754	}
[15033]	755	} else if (!optionNoEli) {
	756	myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
[11981]	757	}
	758	}
	759
[15033]	760	et = getAttributionUrl(e);
	761	jt = getAttributionUrl(j);
	762	if (!Objects.equals(et, jt)) {
	763	if (isBlank(jt)) {
	764	myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
	765	} else if (isNotBlank(et)) {
[15034]	766	String ethttps = et.replace("http:", "https:");
[15033]	767	if (jt.equals(ethttps) \|\| jt.equals(et+"/") \|\| jt.equals(ethttps+"/")) {
	768	myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13518]	769	} else {
[15033]	770	myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13518]	771	}
[15033]	772	} else if (!optionNoEli) {
	773	myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
[11981]	774	}
	775	}
	776
[15033]	777	et = getAttributionText(e);
	778	jt = getAttributionText(j);
	779	if (!Objects.equals(et, jt)) {
	780	if (isBlank(jt)) {
	781	myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
	782	} else if (isNotBlank(et)) {
	783	myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
	784	} else if (!optionNoEli) {
	785	myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
[11981]	786	}
	787	}
	788
[15033]	789	et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
	790	jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
	791	if (!Objects.equals(et, jt)) {
	792	if (isBlank(jt)) {
	793	String t = getType(e);
	794	if ("wms_endpoint".equals(t) \|\| "tms".equals(t)) {
	795	myprintln("+ ELI projections for type "+t+": "+getDescription(j));
	796	} else {
	797	myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
[13592]	798	}
[15033]	799	} else if (isNotBlank(et)) {
	800	if ("EPSG:3857 EPSG:4326".equals(et) \|\| "EPSG:3857".equals(et) \|\| "EPSG:4326".equals(et)) {
	801	myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13538]	802	} else {
[15033]	803	myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13538]	804	}
[15033]	805	} else if (!optionNoEli && !"tms".equals(getType(e))) {
	806	myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
[11981]	807	}
	808	}
[12226]	809
[15033]	810	boolean ed = getDefault(e);
	811	boolean jd = getDefault(j);
	812	if (ed != jd) {
	813	if (!jd) {
	814	myprintln("- Missing JOSM default: "+getDescription(j));
	815	} else if (!optionNoEli) {
	816	myprintln("+ Missing ELI default: "+getDescription(j));
[12226]	817	}
	818	}
[15033]	819	boolean eo = getOverlay(e);
	820	boolean jo = getOverlay(j);
	821	if (eo != jo) {
	822	if (!jo) {
	823	myprintln("- Missing JOSM overlay flag: "+getDescription(j));
	824	} else if (!optionNoEli) {
	825	myprintln("+ Missing ELI overlay flag: "+getDescription(j));
[13536]	826	}
	827	}
[11981]	828	}
[15034]	829	}
	830
	831	void doMismatchingShapes() throws IOException {
[15033]	832	myprintln("* Mismatching shapes: *");
	833	for (String url : josmUrls.keySet()) {
	834	ImageryInfo j = josmUrls.get(url);
	835	int num = 1;
	836	for (Shape shape : getShapes(j)) {
	837	List<Coordinate> p = shape.getPoints();
[15034]	838	if (!p.get(0).equals(p.get(p.size()-1))) {
[15033]	839	myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
[11410]	840	}
[15033]	841	for (int nump = 1; nump < p.size(); ++nump) {
	842	if (Objects.equals(p.get(nump-1), p.get(nump))) {
	843	myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
[11964]	844	}
	845	}
[15033]	846	++num;
[11410]	847	}
	848	}
[15033]	849	for (String url : eliUrls.keySet()) {
	850	JsonObject e = eliUrls.get(url);
	851	int num = 1;
	852	List<Shape> s = null;
[13771]	853	try {
[15033]	854	s = getShapes(e);
	855	for (Shape shape : s) {
	856	List<Coordinate> p = shape.getPoints();
[15034]	857	if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
[15033]	858	myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
[11964]	859	}
[15033]	860	for (int nump = 1; nump < p.size(); ++nump) {
	861	if (Objects.equals(p.get(nump-1), p.get(nump))) {
	862	myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
[13771]	863	}
	864	}
[15033]	865	++num;
[11964]	866	}
[15033]	867	} catch (IllegalArgumentException err) {
	868	String desc = getDescription(e);
	869	myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
[11410]	870	}
[13780]	871	if (s == null \|\| !josmUrls.containsKey(url)) {
[15033]	872	continue;
[11410]	873	}
[15033]	874	ImageryInfo j = josmUrls.get(url);
	875	List<Shape> js = getShapes(j);
	876	if (s.isEmpty() && !js.isEmpty()) {
	877	if (!optionNoEli) {
	878	myprintln("+ No ELI shape: "+getDescription(j));
[11414]	879	}
[15034]	880	} else if (js.isEmpty() && !s.isEmpty()) {
[11415]	881	// don't report boundary like 5 point shapes as difference
[15033]	882	if (s.size() != 1 \|\| s.get(0).getPoints().size() != 5) {
	883	myprintln("- No JOSM shape: "+getDescription(j));
[11415]	884	}
[15034]	885	} else if (s.size() != js.size()) {
[15033]	886	myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
[11413]	887	} else {
[15033]	888	boolean[] edone = new boolean[s.size()];
	889	boolean[] jdone = new boolean[js.size()];
	890	for (int enums = 0; enums < s.size(); ++enums) {
	891	List<Coordinate> ep = s.get(enums).getPoints();
	892	for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
	893	List<Coordinate> jp = js.get(jnums).getPoints();
	894	if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]	895	boolean err = false;
[15034]	896	for (int nump = 0; nump < ep.size() && !err; ++nump) {
[15033]	897	Coordinate ept = ep.get(nump);
	898	Coordinate jpt = jp.get(nump);
[15034]	899	if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 \|\| Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
[15033]	900	err = true;
[11413]	901	}
[15034]	902	if (!err) {
[15033]	903	edone[enums] = true;
	904	jdone[jnums] = true;
	905	break;
[14576]	906	}
[11413]	907	}
	908	}
[11411]	909	}
[15033]	910	for (int enums = 0; enums < s.size(); ++enums) {
	911	List<Coordinate> ep = s.get(enums).getPoints();
	912	for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
	913	List<Coordinate> jp = js.get(jnums).getPoints();
	914	if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]	915	boolean err = false;
[15033]	916	for (int nump = 0; nump < ep.size() && !err; ++nump) {
	917	Coordinate ept = ep.get(nump);
	918	Coordinate jpt = jp.get(nump);
	919	if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 \|\| Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
	920	String numtxt = Integer.toString(enums+1);
	921	if (enums != jnums) {
	922	numtxt += '/' + Integer.toString(jnums+1);
	923	}
	924	myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
	925	break;
[14576]	926	}
	927	}
[15033]	928	edone[enums] = true;
	929	jdone[jnums] = true;
	930	break;
[14576]	931	}
	932	}
	933	}
[15033]	934	for (int enums = 0; enums < s.size(); ++enums) {
	935	List<Coordinate> ep = s.get(enums).getPoints();
	936	for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
	937	List<Coordinate> jp = js.get(jnums).getPoints();
	938	if (!jdone[jnums]) {
	939	String numtxt = Integer.toString(enums+1);
	940	if (enums != jnums) {
	941	numtxt += '/' + Integer.toString(jnums+1);
	942	}
[15034]	943	myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
	944	+ getDescription(j));
[15033]	945	edone[enums] = true;
	946	jdone[jnums] = true;
	947	break;
[14576]	948	}
	949	}
	950	}
[11410]	951	}
	952	}
[15034]	953	}
	954
	955	void doMismatchingIcons() throws IOException {
[15033]	956	myprintln("* Mismatching icons: *");
	957	for (String url : eliUrls.keySet()) {
	958	JsonObject e = eliUrls.get(url);
[11420]	959	if (!josmUrls.containsKey(url)) {
[15033]	960	continue;
[11420]	961	}
[15033]	962	ImageryInfo j = josmUrls.get(url);
	963	String ij = getIcon(j);
	964	String ie = getIcon(e);
	965	boolean ijok = isNotBlank(ij);
	966	boolean ieok = isNotBlank(ie);
	967	if (ijok && !ieok) {
	968	if (!optionNoEli) {
	969	myprintln("+ No ELI icon: "+getDescription(j));
[11420]	970	}
[15033]	971	} else if (!ijok && ieok) {
	972	myprintln("- No JOSM icon: "+getDescription(j));
	973	} else if (ijok && ieok && !Objects.equals(ij, ie) && !(
[14351]	974	(ie.startsWith("https://osmlab.github.io/editor-layer-index/")
	975	\|\| ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
[14293]	976	ij.startsWith("data:"))) {
[15034]	977	String iehttps = ie.replace("http:", "https:");
[15033]	978	if (ij.equals(iehttps)) {
	979	myprintln("+ Different icons: "+getDescription(j));
[14294]	980	} else {
[15033]	981	myprintln("* Different icons: "+getDescription(j));
[14294]	982	}
[11420]	983	}
	984	}
[15034]	985	}
	986
	987	void doMiscellaneousChecks() throws IOException {
[15033]	988	myprintln("* Miscellaneous checks: *");
	989	Map<String, ImageryInfo> josmIds = new HashMap<>();
	990	Collection<String> all = Projections.getAllProjectionCodes();
[13551]	991	DomainValidator dv = DomainValidator.getInstance();
[15033]	992	for (String url : josmUrls.keySet()) {
	993	ImageryInfo j = josmUrls.get(url);
	994	String id = getId(j);
	995	if ("wms".equals(getType(j))) {
	996	String urlLc = url.toLowerCase(Locale.ENGLISH);
	997	if (getProjections(j).isEmpty()) {
	998	myprintln("* WMS without projections: "+getDescription(j));
[13526]	999	} else {
[15033]	1000	List<String> unsupported = new LinkedList<>();
	1001	List<String> old = new LinkedList<>();
	1002	for (String p : getProjectionsUnstripped(j)) {
	1003	if ("CRS:84".equals(p)) {
	1004	if (!urlLc.contains("version=1.3")) {
	1005	myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
[13526]	1006	}
[15033]	1007	} else if (oldproj.containsKey(p)) {
	1008	old.add(p);
	1009	} else if (!all.contains(p) && !ignoreproj.contains(p)) {
	1010	unsupported.add(p);
[13526]	1011	}
	1012	}
[15033]	1013	if (!unsupported.isEmpty()) {
	1014	myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
[13526]	1015	}
[15033]	1016	for (String o : old) {
[15034]	1017	myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
	1018	+ getDescription(j));
[13527]	1019	}
[13526]	1020	}
[15033]	1021	if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
	1022	myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
	1023	} else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
	1024	myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
[13526]	1025	}
[13511]	1026	}
[15033]	1027	List<String> urls = new LinkedList<>();
	1028	if (!"scanex".equals(getType(j))) {
	1029	urls.add(url);
[13532]	1030	}
[15033]	1031	String jt = getPermissionReferenceUrl(j);
	1032	if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
	1033	urls.add(jt);
	1034	jt = getTermsOfUseUrl(j);
	1035	if (isNotBlank(jt))
	1036	urls.add(jt);
	1037	jt = getAttributionUrl(j);
	1038	if (isNotBlank(jt))
	1039	urls.add(jt);
	1040	jt = getIcon(j);
	1041	if (isNotBlank(jt) && !jt.startsWith("data:image/png;base64,"))
	1042	urls.add(jt);
	1043	Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?/.*");
	1044	for (String u : urls) {
	1045	Matcher m = patternU.matcher(u);
	1046	if (!m.matches() \|\| u.matches(".*[ \t]+$")) {
	1047	myprintln("* Strange URL '"+u+"': "+getDescription(j));
	1048	} else {
[15034]	1049	String domain = m.group(1).replaceAll("\\{switch:.*\\}", "x");
[15033]	1050	String port = m.group(2);
	1051	if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
	1052	myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
	1053	else if (":80".equals(port) \|\| ":443".equals(port)) {
	1054	myprintln("* Useless port '"+port+"': "+getDescription(j));
[13554]	1055	}
[13551]	1056	}
	1057	}
	1058
[15033]	1059	if (josmMirrors.containsKey(url)) {
	1060	continue;
[11420]	1061	}
[15033]	1062	if (isBlank(id)) {
	1063	myprintln("* No JOSM-ID: "+getDescription(j));
	1064	} else if (josmIds.containsKey(id)) {
	1065	myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
[11420]	1066	} else {
[15033]	1067	josmIds.put(id, j);
[11420]	1068	}
[15033]	1069	String d = getDate(j);
	1070	if (isNotBlank(d)) {
	1071	Pattern patternD = Pattern.compile("^(-\|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-\|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
	1072	Matcher m = patternD.matcher(d);
	1073	if (!m.matches()) {
	1074	myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
[11572]	1075	} else {
	1076	try {
[15033]	1077	Date first = verifyDate(m.group(2), m.group(4), m.group(6));
	1078	Date second = verifyDate(m.group(9), m.group(11), m.group(13));
	1079	if (second.compareTo(first) < 0) {
	1080	myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
[11572]	1081	}
[15034]	1082	} catch (Exception e) {
[15033]	1083	myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
[11572]	1084	}
	1085	}
[11603]	1086	}
[15033]	1087	if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
	1088	myprintln("* Attribution link without text: "+getDescription(j));
[12261]	1089	}
[15033]	1090	if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
	1091	myprintln("* Logo link without image: "+getDescription(j));
[12261]	1092	}
[15033]	1093	if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
	1094	myprintln("* Terms of Use text without link: "+getDescription(j));
[12261]	1095	}
[15033]	1096	List<Shape> js = getShapes(j);
	1097	if (!js.isEmpty()) {
	1098	double minlat = 1000;
	1099	double minlon = 1000;
	1100	double maxlat = -1000;
	1101	double maxlon = -1000;
	1102	for (Shape s: js) {
	1103	for (Coordinate p: s.getPoints()) {
	1104	double lat = p.getLat();
	1105	double lon = p.getLon();
[15034]	1106	if (lat > maxlat) maxlat = lat;
	1107	if (lon > maxlon) maxlon = lon;
	1108	if (lat < minlat) minlat = lat;
	1109	if (lon < minlon) minlon = lon;
[11422]	1110	}
	1111	}
[15033]	1112	ImageryBounds b = j.getBounds();
	1113	if (b.getMinLat() != minlat \|\| b.getMinLon() != minlon \|\| b.getMaxLat() != maxlat \|\| b.getMaxLon() != maxlon) {
	1114	myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
[15034]	1115	+ ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
	1116	+ getDescription(j));
[11422]	1117	}
	1118	}
[15033]	1119	List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
	1120	String cat = getCategory(j);
	1121	if (isBlank(cat)) {
	1122	myprintln("* No category: "+getDescription(j));
	1123	} else if (!knownCategories.contains(cat)) {
	1124	myprintln("* Strange category "+cat+": "+getDescription(j));
[13792]	1125	}
[11420]	1126	}
[7726]	1127	}
[9667]	1128
[15033]	1129	/*
[7726]	1130	* Utility functions that allow uniform access for both ImageryInfo and JsonObject.
	1131	*/
[15033]	1132
[7726]	1133	static String getUrl(Object e) {
[15033]	1134	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
	1135	return ((Map<String, JsonObject>) e).get("properties").getString("url");
[7726]	1136	}
[15033]	1137
[12242]	1138	static String getUrlStripped(Object e) {
[15034]	1139	return getUrl(e).replaceAll("\\?(apikey\|access_token)=.*", "");
[12242]	1140	}
[15033]	1141
[11572]	1142	static String getDate(Object e) {
[15033]	1143	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
	1144	JsonObject p = ((Map<String, JsonObject>) e).get("properties");
	1145	String start = p.containsKey("start_date") ? p.getString("start_date") : "";
	1146	String end = p.containsKey("end_date") ? p.getString("end_date") : "";
[15034]	1147	if (!start.isEmpty() && !end.isEmpty())
[15033]	1148	return start+";"+end;
[15034]	1149	else if (!start.isEmpty())
[15033]	1150	return start+";-";
[15034]	1151	else if (!end.isEmpty())
[15033]	1152	return "-;"+end;
	1153	return "";
[11572]	1154	}
[15033]	1155
	1156	static Date verifyDate(String year, String month, String day) throws ParseException {
	1157	String date;
[15034]	1158	if (year == null) {
[15033]	1159	date = "3000-01-01";
[11854]	1160	} else {
[15033]	1161	date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
[11854]	1162	}
[15033]	1163	SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
	1164	df.setLenient(false);
	1165	return df.parse(date);
[11572]	1166	}
[15033]	1167
[11420]	1168	static String getId(Object e) {
[15033]	1169	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
	1170	return ((Map<String, JsonObject>) e).get("properties").getString("id");
[11420]	1171	}
[15033]	1172
[7726]	1173	static String getName(Object e) {
[15033]	1174	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
	1175	return ((Map<String, JsonObject>) e).get("properties").getString("name");
[7726]	1176	}
[15033]	1177
	1178	static List<ImageryInfo> getMirrors(Object e) {
	1179	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
	1180	return Collections.emptyList();
[11967]	1181	}
[15033]	1182
	1183	static List<String> getProjections(Object e) {
	1184	List<String> r = new ArrayList<>();
	1185	List<String> u = getProjectionsUnstripped(e);
	1186	if (u != null) {
	1187	for (String p : u) {
	1188	if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
	1189	r.add(p);
[13530]	1190	}
	1191	}
	1192	}
[15033]	1193	return r;
[13530]	1194	}
[15033]	1195
	1196	static List<String> getProjectionsUnstripped(Object e) {
	1197	List<String> r = null;
[11975]	1198	if (e instanceof ImageryInfo) {
[15033]	1199	r = ((ImageryInfo) e).getServerProjections();
[11975]	1200	} else {
[15033]	1201	JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
	1202	if (s != null) {
	1203	r = new ArrayList<>();
	1204	for (JsonValue p : s.asJsonArray()) {
	1205	r.add(((JsonString) p).getString());
[13530]	1206	}
[11981]	1207	}
[11975]	1208	}
[15033]	1209	return r != null ? r : Collections.emptyList();
[11975]	1210	}
[15033]	1211
[11410]	1212	static List<Shape> getShapes(Object e) {
	1213	if (e instanceof ImageryInfo) {
[15033]	1214	ImageryBounds bounds = ((ImageryInfo) e).getBounds();
[15034]	1215	if (bounds != null) {
[15033]	1216	return bounds.getShapes();
[11411]	1217	}
[15033]	1218	return Collections.emptyList();
[11410]	1219	}
[15033]	1220	JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
	1221	if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
	1222	JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
	1223	List<Shape> l = new ArrayList<>();
	1224	for (JsonValue shapes: poly) {
	1225	Shape s = new Shape();
	1226	for (JsonValue point: shapes.asJsonArray()) {
	1227	String lon = point.asJsonArray().getJsonNumber(0).toString();
	1228	String lat = point.asJsonArray().getJsonNumber(1).toString();
	1229	s.addPoint(lat, lon);
[11410]	1230	}
[15033]	1231	l.add(s);
[11410]	1232	}
[15033]	1233	return l;
[11410]	1234	}
[15033]	1235	return Collections.emptyList();
[11410]	1236	}
[15033]	1237
[7726]	1238	static String getType(Object e) {
[15033]	1239	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
	1240	return ((Map<String, JsonObject>) e).get("properties").getString("type");
[7726]	1241	}
[15033]	1242
[7726]	1243	static Integer getMinZoom(Object e) {
	1244	if (e instanceof ImageryInfo) {
[15033]	1245	int mz = ((ImageryInfo) e).getMinZoom();
	1246	return mz == 0 ? null : mz;
[7726]	1247	} else {
[15033]	1248	JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
	1249	if (num == null) return null;
	1250	return num.intValue();
[7726]	1251	}
	1252	}
[15033]	1253
[7726]	1254	static Integer getMaxZoom(Object e) {
	1255	if (e instanceof ImageryInfo) {
[15033]	1256	int mz = ((ImageryInfo) e).getMaxZoom();
	1257	return mz == 0 ? null : mz;
[7726]	1258	} else {
[15033]	1259	JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
	1260	if (num == null) return null;
	1261	return num.intValue();
[7726]	1262	}
	1263	}
[15033]	1264
[7726]	1265	static String getCountryCode(Object e) {
[15033]	1266	if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
	1267	return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
[7726]	1268	}
[15033]	1269
[9505]	1270	static String getQuality(Object e) {
[15033]	1271	if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
	1272	return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
	1273	&& ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
[9505]	1274	}
[15033]	1275
	1276	static boolean getOverlay(Object e) {
	1277	if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
	1278	return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
	1279	&& ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
[13536]	1280	}
[15033]	1281
[11420]	1282	static String getIcon(Object e) {
[15033]	1283	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
	1284	return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
[11420]	1285	}
[15033]	1286
[11975]	1287	static String getAttributionText(Object e) {
[15033]	1288	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
	1289	try {
	1290	return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
	1291	} catch (NullPointerException ex) {
	1292	return null;
	1293	}
[11975]	1294	}
[15033]	1295
[11975]	1296	static String getAttributionUrl(Object e) {
[15033]	1297	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
	1298	try {
	1299	return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
	1300	} catch (NullPointerException ex) {
	1301	return null;
	1302	}
[11975]	1303	}
[15033]	1304
[11975]	1305	static String getTermsOfUseText(Object e) {
[15033]	1306	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
	1307	return null;
[11975]	1308	}
[15033]	1309
[11975]	1310	static String getTermsOfUseUrl(Object e) {
[15033]	1311	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
	1312	return null;
[11975]	1313	}
[15033]	1314
[13792]	1315	static String getCategory(Object e) {
	1316	if (e instanceof ImageryInfo) {
[15033]	1317	return ((ImageryInfo) e).getImageryCategoryOriginalString();
[13792]	1318	}
[15033]	1319	return null;
[13792]	1320	}
[15033]	1321
[12261]	1322	static String getLogoImage(Object e) {
[15033]	1323	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
	1324	return null;
[12261]	1325	}
[15033]	1326
[12261]	1327	static String getLogoUrl(Object e) {
[15033]	1328	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
	1329	return null;
[12261]	1330	}
[15033]	1331
[11975]	1332	static String getPermissionReferenceUrl(Object e) {
[15033]	1333	if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
	1334	return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
[11975]	1335	}
[15033]	1336
[15034]	1337	static Map<String, String> getDescriptions(Object e) {
	1338	Map<String, String> res = new HashMap<String, String>();
[11975]	1339	if (e instanceof ImageryInfo) {
[15033]	1340	String a = ((ImageryInfo) e).getDescription();
	1341	if (a != null) res.put("en", a);
[11975]	1342	} else {
[15033]	1343	String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
[15034]	1344	if (a != null) res.put("en", a.replaceAll("''", "'"));
[11975]	1345	}
[15033]	1346	return res;
[11975]	1347	}
[15033]	1348
	1349	static boolean getValidGeoreference(Object e) {
	1350	if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
	1351	return false;
[11975]	1352	}
[15033]	1353
	1354	static boolean getDefault(Object e) {
	1355	if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
	1356	return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
[12226]	1357	}
[15033]	1358
[7726]	1359	String getDescription(Object o) {
[15033]	1360	String url = getUrl(o);
	1361	String cc = getCountryCode(o);
[7726]	1362	if (cc == null) {
[15033]	1363	ImageryInfo j = josmUrls.get(url);
	1364	if (j != null) cc = getCountryCode(j);
[7726]	1365	if (cc == null) {
[15033]	1366	JsonObject e = eliUrls.get(url);
	1367	if (e != null) cc = getCountryCode(e);
[7726]	1368	}
	1369	}
	1370	if (cc == null) {
[15033]	1371	cc = "";
[7726]	1372	} else {
[15033]	1373	cc = "["+cc+"] ";
[7726]	1374	}
[15033]	1375	String d = cc + getName(o) + " - " + getUrl(o);
	1376	if (optionShorten) {
	1377	if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
[7726]	1378	}
[15033]	1379	return d;
[7726]	1380	}
	1381	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: