Context Navigation

source: josm/trunk/scripts/sync_editor_imagery_index.groovy@ 9552

Last change on this file since 9552 was 9518, checked in by stoecker, 8 years ago
see #12313 - fix zoomlevel diff for minimum zoom 0
Property svn:eol-style set to `native`
File size: 18.0 KB

Rev	Line
[7726]	1	// License: GPL. For details, see LICENSE file.
	2	/**
	3	* Compare and analyse the differences of the editor imagery index and the JOSM imagery list.
	4	* The goal is to keep both lists in sync.
	5	*
	6	* The editor imagery index project (https://github.com/osmlab/editor-imagery-index)
	7	* provides also a version in the JOSM format, but the JSON is the original source
	8	* format, so we read that.
	9	*
	10	* How to run:
	11	* -----------
	12	*
	13	* Main JOSM binary needs to be in classpath, e.g.
	14	*
	15	* $ groovy -cp ../dist/josm-custom.jar sync_editor-imagery-index.groovy
	16	*
	17	* Add option "-h" to show the available command line flags.
	18	*/
	19	import java.io.FileReader
	20	import java.util.List
	21
	22	import javax.json.Json
	23	import javax.json.JsonArray
	24	import javax.json.JsonObject
	25	import javax.json.JsonReader
	26
	27	import org.openstreetmap.josm.io.imagery.ImageryReader
	28	import org.openstreetmap.josm.data.imagery.ImageryInfo
	29	import org.openstreetmap.josm.tools.Utils
	30
	31	class sync_editor_imagery_index {
	32
	33	List<ImageryInfo> josmEntries;
	34	JsonArray eiiEntries;
	35
	36	def eiiUrls = new HashMap<String, JsonObject>()
	37	def josmUrls = new HashMap<String, ImageryInfo>()
	38
	39	static String eiiInputFile = 'imagery.json'
	40	static String josmInputFile = 'maps.xml'
[9505]	41	static FileWriter outputFile = null
	42	static BufferedWriter outputStream = null
	43	static int skipCount = 0;
	44	static def skipEntries = [:]
	45
[7726]	46	static def options
	47
	48	/**
	49	* Main method.
	50	*/
	51	static main(def args) {
	52	parse_command_line_arguments(args)
	53	def script = new sync_editor_imagery_index()
[9505]	54	script.loadSkip()
[7726]	55	script.loadJosmEntries()
	56	script.loadEIIEntries()
	57	script.checkInOneButNotTheOther()
	58	script.checkCommonEntries()
[9505]	59	if(outputStream != null) {
	60	outputStream.close();
	61	}
	62	if(outputFile != null) {
	63	outputFile.close();
	64	}
[7726]	65	}
	66
	67	/**
	68	* Parse command line arguments.
	69	*/
	70	static void parse_command_line_arguments(args) {
	71	def cli = new CliBuilder()
[9505]	72	cli.o(longOpt:'output', args:1, argName: "output", "Output file, - prints to stdout (default: -)")
	73	cli.e(longOpt:'eii_input', args:1, argName:"eii_input", "Input file for the editor imagery index (json). Default is $eiiInputFile (current directory).")
	74	cli.j(longOpt:'josm_input', args:1, argName:"josm_input", "Input file for the JOSM imagery list (xml). Default is $josmInputFile (current directory).")
[7726]	75	cli.s(longOpt:'shorten', "shorten the output, so it is easier to read in a console window")
[9505]	76	cli.n(longOpt:'noskip', argName:"noskip", "don't skip known entries")
	77	cli.m(longOpt:'nomissingeii', argName:"nomissingeii", "don't show missing editor imagery index entries")
[7726]	78	cli.h(longOpt:'help', "show this help")
	79	options = cli.parse(args)
	80
	81	if (options.h) {
	82	cli.usage()
	83	System.exit(0)
	84	}
	85	if (options.eii_input) {
	86	eiiInputFile = options.eii_input
	87	}
	88	if (options.josm_input) {
	89	josmInputFile = options.josm_input
	90	}
[9505]	91	if (options.output && options.output != "-") {
	92	outputFile = new FileWriter(options.output)
	93	outputStream = new BufferedWriter(outputFile)
	94	}
[7726]	95	}
	96
[9505]	97	void loadSkip() {
	98	if (options.noskip)
	99	return;
	100	skipEntries["+++ EII-URL is not unique: http://geolittoral.application.equipement.gouv.fr/wms/metropole?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=ortholittorale&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
[9515]	101	skipEntries[" Streets NRW Geofabrik.de - http://tools.geofabrik.de/osmi/view/strassennrw/wxs?REQUEST=GetMap&SERVICE=wms&VERSION=1.1.1&FORMAT=image/png&SRS={proj}&STYLES=&LAYERS=unzugeordnete_strassen,kreisstrassen_ast,kreisstrassen,landesstrassen_ast,landesstrassen,bundesstrassen_ast,bundesstrassen,autobahnen_ast,autobahnen,endpunkte&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	102	skipEntries[" Czech UHUL:ORTOFOTO - http://geoportal2.uhul.cz/cgi-bin/oprl.asp?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&SRS={proj}&LAYERS=Ortofoto_cb&STYLES=default&FORMAT=image/jpeg&TRANSPARENT=TRUE&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	103	skipEntries[" Czech ÚHUL:ORTOFOTO tiles proxy - http://osm-{switch:a,b,c}.zby.cz/tiles_uhul.php/{zoom}/{x}/{y}.jpg"] = 1
	104	skipEntries[" [EE] Estonia Basemap (Maaamet) - http://kaart.maaamet.ee/wms/alus-geo?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=pohi_vr2&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	105	skipEntries[" [EE] Estonia Forestry (Maaamet) - http://kaart.maaamet.ee/wms/alus-geo?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=cir_ngr&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	106	skipEntries[" [EE] Estonia Hillshading (Maaamet) - http://kaart.maaamet.ee/wms/alus-geo?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=reljeef&STYLES=&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	107	skipEntries[" [EE] Estonia Ortho (Maaamet) - http://kaart.maaamet.ee/wms/alus-geo?VERSION=1.1.1&REQUEST=GetMap&LAYERS=of10000&SRS={proj}&FORMAT=image/jpeg&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	108	skipEntries[" Hamburg (DK5) - http://gateway.hamburg.de/OGCFassade/HH_WMS_Geobasisdaten.aspx?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&LAYERS=1&STYLES=&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
	109	skipEntries[" Hamburg (40 cm) - http://gateway.hamburg.de/OGCFassade/HH_WMS_DOP40.aspx?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&LAYERS=0&STYLES=&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 1
[9505]	110	skipEntries[" name differs: http://wms.openstreetmap.fr/tms/1.0.0/tours_2013/{zoom}/{x}/{y}"] = 3
	111	skipEntries[" name differs: http://wms.openstreetmap.fr/tms/1.0.0/tours/{zoom}/{x}/{y}"] = 3
	112	skipEntries[" name differs: https://secure.erlangen.de/arcgiser/services/Luftbilder2011/MapServer/WmsServer?FORMAT=image/bmp&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&LAYERS=Erlangen_ratio10_5cm_gk4.jp2&STYLES=&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 3
	113	skipEntries[" name differs: http://wms.openstreetmap.fr/tms/1.0.0/iomhaiti/{zoom}/{x}/{y}"] = 3
	114	skipEntries[" name differs: http://{switch:a,b,c}.layers.openstreetmap.fr/bano/{zoom}/{x}/{y}.png"] = 3
	115	skipEntries[" name differs: http://ooc.openstreetmap.org/os1/{zoom}/{x}/{y}.jpg"] = 3
	116	skipEntries[" name differs: http://www.gisnet.lv/cgi-bin/osm_latvia?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=piekraste&SRS={proj}&WIDTH={width}&height={height}&BBOX={bbox}"] = 3
	117	skipEntries[" name differs: http://tms.cadastre.openstreetmap.fr/*/tout/{z}/{x}/{y}.png"] = 3
	118	skipEntries[" name differs: http://{switch:a,b,c}.tiles.mapbox.com/v4/enf.e0b8291e/{zoom}/{x}/{y}.png?access_token=pk.eyJ1Ijoib3BlbnN0cmVldG1hcCIsImEiOiJhNVlHd29ZIn0.ti6wATGDWOmCnCYen-Ip7Q"] = 3
	119	skipEntries[" name differs: http://geo.nls.uk/mapdata2/os/25_inch/scotland_1/{zoom}/{x}/{y}.png"] = 3
	120	skipEntries[" name differs: http://geo.nls.uk/mapdata3/os/6_inch_gb_1900/{zoom}/{x}/{y}.png"] = 3
	121	skipEntries[" maxzoom differs: [DE] Bavaria (2 m) - http://geodaten.bayern.de/ogc/ogc_dop200_oa.cgi?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=adv_dop200c&SRS={proj}&WIDTH={width}&HEIGHT={height}&BBOX={bbox}"] = 3
	122	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries County - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=County&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	123	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries NPWS Reserve - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=NPWSReserve&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	124	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries Parish - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=Parish&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	125	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries Suburb - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=Suburb&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	126	skipEntries[" minzoom differs: [AU] LPI NSW Imagery - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Imagery/MapServer/tile/{zoom}/{y}/{x}"] = 3
	127	skipEntries[" minzoom differs: [AU] LPI NSW Topographic Map - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Topo_Map/MapServer/tile/{zoom}/{y}/{x}"] = 3
	128	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries State Forest - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=StateForest&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	129	skipEntries[" minzoom differs: [AU] LPI NSW Administrative Boundaries LGA - http://maps.six.nsw.gov.au/arcgis/services/public/NSW_Administrative_Boundaries/MapServer/WMSServer?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&CRS={proj}&BBOX={bbox}&WIDTH={width}&HEIGHT={height}&LAYERS=LocalGovernmentArea&STYLES=&FORMAT=image/png32&DPI=96&MAP_RESOLUTION=96&FORMAT_OPTIONS=dpi:96&TRANSPARENT=TRUE"] = 3
	130	skipEntries[" minzoom differs: [AU] LPI NSW Base Map - http://maps.six.nsw.gov.au/arcgis/rest/services/public/NSW_Base_Map/MapServer/tile/{zoom}/{y}/{x}"] = 3
	131	}
	132
	133	void myprintln(String s) {
	134	if(skipEntries.containsKey(s)) {
	135	skipCount = skipEntries.get(s)
	136	}
	137	if(skipCount) {
	138	skipCount -= 1;
	139	return
	140	}
	141	if(outputStream != null) {
	142	outputStream.write(s);
	143	outputStream.newLine();
	144	} else {
	145	println s;
	146	}
	147	}
	148
[7726]	149	void loadEIIEntries() {
	150	FileReader fr = new FileReader(eiiInputFile)
	151	JsonReader jr = Json.createReader(fr)
	152	eiiEntries = jr.readArray()
	153	jr.close()
	154
	155	for (def e : eiiEntries) {
	156	def url = getUrl(e)
[9505]	157	if (eiiUrls.containsKey(url)) {
	158	myprintln "+++ EII-URL is not unique: "+url
	159	} else {
	160	eiiUrls.put(url, e)
	161	}
[7726]	162	}
[9505]	163	myprintln "* Loaded ${eiiEntries.size()} entries (EII). *"
[7726]	164	}
	165
	166	void loadJosmEntries() {
	167	def reader = new ImageryReader(josmInputFile)
	168	josmEntries = reader.parse()
	169
	170	for (def e : josmEntries) {
	171	def url = getUrl(e)
[9505]	172	if (josmUrls.containsKey(url)) {
	173	myprintln "+++ JOSM-URL is not unique: "+url
	174	} else {
	175	josmUrls.put(url, e)
[7726]	176	}
	177	}
[9505]	178	myprintln "* Loaded ${josmEntries.size()} entries (JOSM). *"
[7726]	179	}
	180
	181	List inOneButNotTheOther(Map m1, Map m2) {
	182	def l = []
	183	for (def url : m1.keySet()) {
	184	if (!m2.containsKey(url)) {
	185	def name = getName(m1.get(url))
	186	l += " "+getDescription(m1.get(url))
	187	}
	188	}
	189	l.sort()
	190	}
	191
	192	void checkInOneButNotTheOther() {
	193	def l1 = inOneButNotTheOther(eiiUrls, josmUrls)
[9505]	194	myprintln "* URLs found in EII but not in JOSM (${l1.size()}): *"
[7726]	195	if (l1.isEmpty()) {
[9505]	196	myprintln " -"
[7726]	197	} else {
[9516]	198	for (def l : l1)
	199	myprintln l
[7726]	200	}
	201
[9505]	202	if (options.nomissingeii)
	203	return
[7726]	204	def l2 = inOneButNotTheOther(josmUrls, eiiUrls)
[9505]	205	myprintln "* URLs found in JOSM but not in EII (${l2.size()}): *"
[7726]	206	if (l2.isEmpty()) {
[9505]	207	myprintln " -"
[7726]	208	} else {
[9516]	209	for (def l : l2)
	210	myprintln l
[7726]	211	}
	212	}
	213
	214	void checkCommonEntries() {
[9505]	215	myprintln "* Same URL, but different name: *"
[7726]	216	for (def url : eiiUrls.keySet()) {
	217	def e = eiiUrls.get(url)
	218	if (!josmUrls.containsKey(url)) continue
	219	def j = josmUrls.get(url)
	220	if (!getName(e).equals(getName(j))) {
[9505]	221	myprintln " name differs: $url"
	222	myprintln " (IEE): ${getName(e)}"
	223	myprintln " (JOSM): ${getName(j)}"
[7726]	224	}
	225	}
	226
[9505]	227	myprintln "* Same URL, but different type: *"
[7726]	228	for (def url : eiiUrls.keySet()) {
	229	def e = eiiUrls.get(url)
	230	if (!josmUrls.containsKey(url)) continue
	231	def j = josmUrls.get(url)
	232	if (!getType(e).equals(getType(j))) {
[9505]	233	myprintln " type differs: ${getName(j)} - $url"
	234	myprintln " (IEE): ${getType(e)}"
	235	myprintln " (JOSM): ${getType(j)}"
[7726]	236	}
	237	}
	238
[9505]	239	myprintln "* Same URL, but different zoom bounds: *"
[7726]	240	for (def url : eiiUrls.keySet()) {
	241	def e = eiiUrls.get(url)
	242	if (!josmUrls.containsKey(url)) continue
	243	def j = josmUrls.get(url)
	244
	245	Integer eMinZoom = getMinZoom(e)
	246	Integer jMinZoom = getMinZoom(j)
[9518]	247	if (eMinZoom != jMinZoom && !(eMinZoom == 0 && jMinZoom == null)) {
[9505]	248	myprintln " minzoom differs: ${getDescription(j)}"
	249	myprintln " (IEE): ${eMinZoom}"
	250	myprintln " (JOSM): ${jMinZoom}"
[7726]	251	}
	252	Integer eMaxZoom = getMaxZoom(e)
	253	Integer jMaxZoom = getMaxZoom(j)
	254	if (eMaxZoom != jMaxZoom) {
[9505]	255	myprintln " maxzoom differs: ${getDescription(j)}"
	256	myprintln " (IEE): ${eMaxZoom}"
	257	myprintln " (JOSM): ${jMaxZoom}"
[7726]	258	}
	259	}
	260
[9505]	261	myprintln "* Same URL, but different country code: *"
[7726]	262	for (def url : eiiUrls.keySet()) {
	263	def e = eiiUrls.get(url)
	264	if (!josmUrls.containsKey(url)) continue
	265	def j = josmUrls.get(url)
	266	if (!getCountryCode(e).equals(getCountryCode(j))) {
[9505]	267	myprintln " country code differs: ${getDescription(j)}"
	268	myprintln " (IEE): ${getCountryCode(e)}"
	269	myprintln " (JOSM): ${getCountryCode(j)}"
[7726]	270	}
	271	}
[9505]	272	myprintln "* Same URL, but different quality: *"
	273	for (def url : eiiUrls.keySet()) {
	274	def e = eiiUrls.get(url)
[9515]	275	if (!josmUrls.containsKey(url)) {
	276	def q = getQuality(e)
	277	if("best".equals(q)) {
	278	myprintln " quality best entry not in JOSM for ${getDescription(e)}"
	279	}
	280	continue
	281	}
[9505]	282	def j = josmUrls.get(url)
	283	if (!getQuality(e).equals(getQuality(j))) {
	284	myprintln " quality differs: ${getDescription(j)}"
	285	myprintln " (IEE): ${getQuality(e)}"
	286	myprintln " (JOSM): ${getQuality(j)}"
	287	}
	288	}
[7726]	289	}
	290
	291	/**
	292	* Utility functions that allow uniform access for both ImageryInfo and JsonObject.
	293	*/
	294	static String getUrl(Object e) {
	295	if (e instanceof ImageryInfo) return e.url
	296	return e.getString("url")
	297	}
	298	static String getName(Object e) {
	299	if (e instanceof ImageryInfo) return e.name
	300	return e.getString("name")
	301	}
	302	static String getType(Object e) {
	303	if (e instanceof ImageryInfo) return e.getImageryType().getTypeString()
	304	return e.getString("type")
	305	}
	306	static Integer getMinZoom(Object e) {
	307	if (e instanceof ImageryInfo) {
	308	int mz = e.getMinZoom()
	309	return mz == 0 ? null : mz
	310	} else {
	311	def ext = e.getJsonObject("extent")
	312	if (ext == null) return null
	313	def num = ext.getJsonNumber("min_zoom")
	314	if (num == null) return null
	315	return num.intValue()
	316	}
	317	}
	318	static Integer getMaxZoom(Object e) {
	319	if (e instanceof ImageryInfo) {
	320	int mz = e.getMaxZoom()
	321	return mz == 0 ? null : mz
	322	} else {
	323	def ext = e.getJsonObject("extent")
	324	if (ext == null) return null
	325	def num = ext.getJsonNumber("max_zoom")
	326	if (num == null) return null
	327	return num.intValue()
	328	}
	329	}
	330	static String getCountryCode(Object e) {
	331	if (e instanceof ImageryInfo) return "".equals(e.getCountryCode()) ? null : e.getCountryCode()
	332	return e.getString("country_code", null)
	333	}
[9505]	334	static String getQuality(Object e) {
	335	//if (e instanceof ImageryInfo) return "".equals(e.getQuality()) ? null : e.getQuality()
	336	if (e instanceof ImageryInfo) return null
	337	return e.get("best") ? "best" : null
	338	}
[7726]	339	String getDescription(Object o) {
	340	def url = getUrl(o)
	341	def cc = getCountryCode(o)
	342	if (cc == null) {
	343	def j = josmUrls.get(url)
	344	if (j != null) cc = getCountryCode(j)
	345	if (cc == null) {
	346	def e = eiiUrls.get(url)
	347	if (e != null) cc = getCountryCode(e)
	348	}
	349	}
	350	if (cc == null) {
	351	cc = ''
	352	} else {
	353	cc = "[$cc] "
	354	}
	355	def d = cc + getName(o) + " - " + getUrl(o)
	356	if (options.shorten) {
	357	def MAXLEN = 140
	358	if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "..."
	359	}
	360	return d
	361	}
	362
	363	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: