source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/Addresses.java@ 14906

Last change on this file since 14906 was 14906, checked in by GerdP, 5 years ago

fix #17475: Validator should mark obsolete type=associatedStreet relations
  • Property svn:eol-style set to native
File size: 20.9 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.util.ArrayList;
8import java.util.Arrays;
9import java.util.Collection;
10import java.util.HashMap;
11import java.util.HashSet;
12import java.util.List;
13import java.util.Locale;
14import java.util.Map;
15import java.util.Map.Entry;
16import java.util.Objects;
17import java.util.Set;
18import java.util.stream.Collectors;
19import java.util.stream.Stream;
20
21import org.openstreetmap.josm.command.Command;
22import org.openstreetmap.josm.command.DeleteCommand;
23import org.openstreetmap.josm.data.coor.EastNorth;
24import org.openstreetmap.josm.data.coor.LatLon;
25import org.openstreetmap.josm.data.osm.Node;
26import org.openstreetmap.josm.data.osm.OsmPrimitive;
27import org.openstreetmap.josm.data.osm.Relation;
28import org.openstreetmap.josm.data.osm.RelationMember;
29import org.openstreetmap.josm.data.osm.TagMap;
30import org.openstreetmap.josm.data.osm.Way;
31import org.openstreetmap.josm.data.preferences.DoubleProperty;
32import org.openstreetmap.josm.data.validation.Severity;
33import org.openstreetmap.josm.data.validation.Test;
34import org.openstreetmap.josm.data.validation.TestError;
35import org.openstreetmap.josm.tools.Geometry;
36import org.openstreetmap.josm.tools.Logging;
37import org.openstreetmap.josm.tools.Pair;
38import org.openstreetmap.josm.tools.SubclassFilteredCollection;
39import org.openstreetmap.josm.tools.Territories;
40import org.openstreetmap.josm.tools.Utils;
41
42/**
43 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
44 * @since 5644
45 */
46public class Addresses extends Test {
47
48 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
49 protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
50 protected static final int MULTIPLE_STREET_NAMES = 2603;
51 protected static final int MULTIPLE_STREET_RELATIONS = 2604;
52 protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
53 protected static final int OBSOLETE_RELATION = 2606;
54
55 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
56 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
57
58 // CHECKSTYLE.OFF: SingleSpaceSeparator
59 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber";
60 protected static final String ADDR_INTERPOLATION = "addr:interpolation";
61 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
62 protected static final String ADDR_PLACE = "addr:place";
63 protected static final String ADDR_STREET = "addr:street";
64 protected static final String ADDR_CITY = "addr:city";
65 protected static final String ADDR_UNIT = "addr:unit";
66 protected static final String ADDR_FLATS = "addr:flats";
67 protected static final String ADDR_HOUSE_NAME = "addr:housename";
68 protected static final String ADDR_POSTCODE = "addr:postcode";
69 protected static final String ASSOCIATED_STREET = "associatedStreet";
70 // CHECKSTYLE.ON: SingleSpaceSeparator
71
72 private Map<String, Collection<OsmPrimitive>> knownAddresses;
73 private Set<String> ignoredAddresses;
74
75
76 /**
77 * Constructor
78 */
79 public Addresses() {
80 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
81 }
82
83 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
84 final List<Relation> list = p.referrers(Relation.class)
85 .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
86 .collect(Collectors.toList());
87 if (list.size() > 1) {
88 Severity level;
89 // warning level only if several relations have different names, see #10945
90 final String name = list.get(0).get("name");
91 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
92 level = Severity.WARNING;
93 } else {
94 level = Severity.OTHER;
95 }
96 List<OsmPrimitive> errorList = new ArrayList<>(list);
97 errorList.add(0, p);
98 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
99 .message(tr("Multiple associatedStreet relations"))
100 .primitives(errorList)
101 .build());
102 }
103 return list;
104 }
105
106 protected void checkHouseNumbersWithoutStreet(OsmPrimitive p) {
107 List<Relation> associatedStreets = getAndCheckAssociatedStreets(p);
108 // Find house number without proper location
109 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
110 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)) {
111 for (Relation r : associatedStreets) {
112 if (r.hasTag("type", ASSOCIATED_STREET)) {
113 return;
114 }
115 }
116 if (p.referrers(Way.class).anyMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
117 return;
118 }
119 // No street found
120 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
121 .message(tr("House number without street"))
122 .primitives(p)
123 .build());
124 }
125 }
126
127 static boolean isPOI(OsmPrimitive p) {
128 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name");
129 }
130
131 static boolean hasAddress(OsmPrimitive p) {
132 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
133 }
134
135 /**
136 * adds the OsmPrimitive to the address map if it complies to the restrictions
137 * @param p OsmPrimitive that has an address
138 */
139 private void collectAddress(OsmPrimitive p) {
140 if (!isPOI(p)) {
141 String simplifiedAddress = getSimplifiedAddress(p);
142 if (!ignoredAddresses.contains(simplifiedAddress)) {
143 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
144 }
145 }
146 }
147
148 protected void initAddressMap(OsmPrimitive primitive) {
149 knownAddresses = new HashMap<>();
150 ignoredAddresses = new HashSet<>();
151 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
152 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
153 for (OsmPrimitive r : p.getReferrers()) {
154 if (hasAddress(r)) {
155 // ignore addresses of buildings that are connected to addr:unit nodes
156 // it's quite reasonable that there are more buildings with this address
157 String simplifiedAddress = getSimplifiedAddress(r);
158 if (!ignoredAddresses.contains(simplifiedAddress)) {
159 ignoredAddresses.add(simplifiedAddress);
160 } else if (knownAddresses.containsKey(simplifiedAddress)) {
161 knownAddresses.remove(simplifiedAddress);
162 }
163 }
164 }
165 }
166 if (hasAddress(p)) {
167 collectAddress(p);
168 }
169 }
170 }
171
172 @Override
173 public void endTest() {
174 knownAddresses = null;
175 ignoredAddresses = null;
176 super.endTest();
177 }
178
179 protected void checkForDuplicate(OsmPrimitive p) {
180 if (knownAddresses == null) {
181 initAddressMap(p);
182 }
183 if (!isPOI(p) && hasAddress(p)) {
184 String simplifiedAddress = getSimplifiedAddress(p);
185 if (ignoredAddresses.contains(simplifiedAddress)) {
186 return;
187 }
188 if (knownAddresses.containsKey(simplifiedAddress)) {
189 double maxDistance = MAX_DUPLICATE_DISTANCE.get();
190 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
191 if (p == p2) {
192 continue;
193 }
194 Severity severityLevel;
195 String city1 = p.get(ADDR_CITY);
196 String city2 = p2.get(ADDR_CITY);
197 double distance = getDistance(p, p2);
198 if (city1 != null && city2 != null) {
199 if (city1.equals(city2)) {
200 if (!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
201 severityLevel = Severity.WARNING;
202 } else {
203 // address including city identical but postcode differs
204 // most likely perfectly fine
205 severityLevel = Severity.OTHER;
206 }
207 } else {
208 // address differs only by city - notify if very close, otherwise ignore
209 if (distance < maxDistance) {
210 severityLevel = Severity.OTHER;
211 } else {
212 continue;
213 }
214 }
215 } else {
216 // at least one address has no city specified
217 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
218 // address including postcode identical
219 severityLevel = Severity.WARNING;
220 } else {
221 // city/postcode unclear - warn if very close, otherwise only notify
222 // TODO: get city from surrounding boundaries?
223 if (distance < maxDistance) {
224 severityLevel = Severity.WARNING;
225 } else {
226 severityLevel = Severity.OTHER;
227 }
228 }
229 }
230 errors.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
231 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
232 .primitives(Arrays.asList(p, p2)).build());
233 }
234 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
235 }
236 }
237 }
238
239 static String getSimplifiedAddress(OsmPrimitive p) {
240 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
241 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
242 return Utils.strip(Stream.of(
243 simplifiedStreetName.replaceAll("[ -]", ""),
244 p.get(ADDR_HOUSE_NUMBER),
245 p.get(ADDR_HOUSE_NAME),
246 p.get(ADDR_UNIT),
247 p.get(ADDR_FLATS))
248 .filter(Objects::nonNull)
249 .collect(Collectors.joining(" ")))
250 .toUpperCase(Locale.ENGLISH);
251 }
252
253 @Override
254 public void visit(Node n) {
255 checkHouseNumbersWithoutStreet(n);
256 checkForDuplicate(n);
257 }
258
259 @Override
260 public void visit(Way w) {
261 checkHouseNumbersWithoutStreet(w);
262 checkForDuplicate(w);
263 }
264
265 @Override
266 public void visit(Relation r) {
267 checkHouseNumbersWithoutStreet(r);
268 checkForDuplicate(r);
269 if (r.hasTag("type", ASSOCIATED_STREET)) {
270 checkIfObsolete(r);
271 // Used to count occurrences of each house number in order to find duplicates
272 Map<String, List<OsmPrimitive>> map = new HashMap<>();
273 // Used to detect different street names
274 String relationName = r.get("name");
275 Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
276 // Used to check distance
277 Set<OsmPrimitive> houses = new HashSet<>();
278 Set<Way> street = new HashSet<>();
279 for (RelationMember m : r.getMembers()) {
280 String role = m.getRole();
281 OsmPrimitive p = m.getMember();
282 if ("house".equals(role)) {
283 houses.add(p);
284 String number = p.get(ADDR_HOUSE_NUMBER);
285 if (number != null) {
286 number = number.trim().toUpperCase(Locale.ENGLISH);
287 List<OsmPrimitive> list = map.get(number);
288 if (list == null) {
289 list = new ArrayList<>();
290 map.put(number, list);
291 }
292 list.add(p);
293 }
294 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
295 if (wrongStreetNames.isEmpty()) {
296 wrongStreetNames.add(r);
297 }
298 wrongStreetNames.add(p);
299 }
300 } else if ("street".equals(role)) {
301 if (p instanceof Way) {
302 street.add((Way) p);
303 }
304 if (relationName != null && p.hasTagDifferent("name", relationName)) {
305 if (wrongStreetNames.isEmpty()) {
306 wrongStreetNames.add(r);
307 }
308 wrongStreetNames.add(p);
309 }
310 }
311 }
312 // Report duplicate house numbers
313 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
314 List<OsmPrimitive> list = entry.getValue();
315 if (list.size() > 1) {
316 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
317 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
318 .primitives(list)
319 .build());
320 }
321 }
322 // Report wrong street names
323 if (!wrongStreetNames.isEmpty()) {
324 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
325 .message(tr("Multiple street names in relation"))
326 .primitives(wrongStreetNames)
327 .build());
328 }
329 // Report addresses too far away
330 if (!street.isEmpty()) {
331 for (OsmPrimitive house : houses) {
332 if (house.isUsable()) {
333 checkDistance(house, street);
334 }
335 }
336 }
337 }
338 }
339
340 /**
341 * returns rough distance between two OsmPrimitives
342 * @param a primitive a
343 * @param b primitive b
344 * @return distance of center of bounding boxes in meters
345 */
346 static double getDistance(OsmPrimitive a, OsmPrimitive b) {
347 LatLon centerA = a.getBBox().getCenter();
348 LatLon centerB = b.getBBox().getCenter();
349 return (centerA.greatCircleDistance(centerB));
350 }
351
352 protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
353 EastNorth centroid;
354 if (house instanceof Node) {
355 centroid = ((Node) house).getEastNorth();
356 } else if (house instanceof Way) {
357 List<Node> nodes = ((Way) house).getNodes();
358 if (house.hasKey(ADDR_INTERPOLATION)) {
359 for (Node n : nodes) {
360 if (n.hasKey(ADDR_HOUSE_NUMBER)) {
361 checkDistance(n, street);
362 }
363 }
364 return;
365 }
366 centroid = Geometry.getCentroid(nodes);
367 } else {
368 return; // TODO handle multipolygon houses ?
369 }
370 if (centroid == null) return; // fix #8305
371 double maxDistance = MAX_STREET_DISTANCE.get();
372 boolean hasIncompleteWays = false;
373 for (Way streetPart : street) {
374 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
375 EastNorth p1 = chunk.a.getEastNorth();
376 EastNorth p2 = chunk.b.getEastNorth();
377 if (p1 != null && p2 != null) {
378 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
379 if (closest.distance(centroid) <= maxDistance) {
380 return;
381 }
382 } else {
383 Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
384 }
385 }
386 if (!hasIncompleteWays && streetPart.isIncomplete()) {
387 hasIncompleteWays = true;
388 }
389 }
390 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
391 if (hasIncompleteWays) return;
392 List<OsmPrimitive> errorList = new ArrayList<>(street);
393 errorList.add(0, house);
394 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
395 .message(tr("House number too far from street"))
396 .primitives(errorList)
397 .build());
398 }
399
400 /**
401 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
402 * are complete and don't contain any information which isn't also tagged on the members.
403 * The strategy is to avoid any false positive.
404 * @param r the relation
405 */
406 private void checkIfObsolete(Relation r) {
407 if (r.isIncomplete())
408 return;
409 /** array of country codes for which the test should be performed. For now, only Germany */
410 String[] countryCodes = {"DE"};
411 TagMap neededtagsForHouse = new TagMap();
412 for (Entry<String, String> tag : r.getKeys().entrySet()) {
413 String key = tag.getKey();
414 if (key.startsWith("name:")) {
415 return; // maybe check if all members have corresponding tags?
416 } else if (key.startsWith("addr:")) {
417 neededtagsForHouse.put(key, tag.getValue());
418 } else {
419 switch (key) {
420 case "name":
421 case "type":
422 case "source":
423 break;
424 default:
425 // unexpected tag in relation
426 return;
427 }
428 }
429 }
430
431 for (RelationMember m : r.getMembers()) {
432 if (m.getMember().isIncomplete() || m.isRelation() || !isInWarnCountry(m, countryCodes))
433 return;
434
435 String role = m.getRole();
436 if ("".equals(role)) {
437 if (m.isWay() && m.getMember().hasKey("highway")) {
438 role = "street";
439 } else if (m.getMember().hasTag("building"))
440 role = "house";
441 }
442 switch (role) {
443 case "house":
444 case "addr:houselink":
445 case "address":
446 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
447 return;
448 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
449 if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
450 return;
451 }
452 break;
453 case "street":
454 if (!m.getMember().hasTag("name") && r.hasTag("name"))
455 return;
456 break;
457 default:
458 // unknown role: don't create auto-fix
459 return;
460 }
461 }
462 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
463 .message(tr("Relation is obsolete"))
464 .primitives(r)
465 .build());
466 }
467
468 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
469 if (countryCodes.length == 0)
470 return true;
471 LatLon center = null;
472
473 if (m.isNode()) {
474 center = m.getNode().getCoor();
475 } else if (m.isWay()) {
476 center = m.getWay().getBBox().getCenter();
477 }
478 if (center == null)
479 return true;
480 for (String country : countryCodes) {
481 if (Territories.isIso3166Code(country, center))
482 return true;
483 }
484 return false;
485 }
486
487 /**
488 * remove obsolete relation.
489 */
490 @Override
491 public Command fixError(TestError testError) {
492 return new DeleteCommand(testError.getPrimitives());
493 }
494
495 @Override
496 public boolean isFixable(TestError testError) {
497 if (!(testError.getTester() instanceof Addresses))
498 return false;
499 return testError.getCode() == OBSOLETE_RELATION;
500 }
501
502}
Note: See TracBrowser for help on using the repository browser.