source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/Addresses.java

Last change on this file was 18922, checked in by taylor.smock, 2 years ago

Fix #23302: Create a preference for address duplicate detection to include buildings and POIs (patch by zyphlar, modified)

Modifications are as follows:

  • Add test case
  • Property svn:eol-style set to native
File size: 24.4 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.util.ArrayList;
8import java.util.Arrays;
9import java.util.Collection;
10import java.util.Collections;
11import java.util.HashMap;
12import java.util.HashSet;
13import java.util.List;
14import java.util.Locale;
15import java.util.Map;
16import java.util.Map.Entry;
17import java.util.Objects;
18import java.util.Set;
19import java.util.stream.Collectors;
20import java.util.stream.Stream;
21
22import javax.swing.JCheckBox;
23import javax.swing.JPanel;
24
25import org.openstreetmap.josm.command.Command;
26import org.openstreetmap.josm.command.DeleteCommand;
27import org.openstreetmap.josm.data.coor.EastNorth;
28import org.openstreetmap.josm.data.coor.ILatLon;
29import org.openstreetmap.josm.data.coor.LatLon;
30import org.openstreetmap.josm.data.osm.Node;
31import org.openstreetmap.josm.data.osm.OsmPrimitive;
32import org.openstreetmap.josm.data.osm.Relation;
33import org.openstreetmap.josm.data.osm.RelationMember;
34import org.openstreetmap.josm.data.osm.TagMap;
35import org.openstreetmap.josm.data.osm.Way;
36import org.openstreetmap.josm.data.preferences.BooleanProperty;
37import org.openstreetmap.josm.data.preferences.DoubleProperty;
38import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
39import org.openstreetmap.josm.data.validation.Severity;
40import org.openstreetmap.josm.data.validation.Test;
41import org.openstreetmap.josm.data.validation.TestError;
42import org.openstreetmap.josm.gui.progress.ProgressMonitor;
43import org.openstreetmap.josm.tools.GBC;
44import org.openstreetmap.josm.tools.Geometry;
45import org.openstreetmap.josm.tools.Logging;
46import org.openstreetmap.josm.tools.Pair;
47import org.openstreetmap.josm.tools.SubclassFilteredCollection;
48import org.openstreetmap.josm.tools.Territories;
49import org.openstreetmap.josm.tools.Utils;
50
51/**
52 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
53 * @since 5644
54 */
55public class Addresses extends Test {
56
57 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
58 protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
59 protected static final int MULTIPLE_STREET_NAMES = 2603;
60 protected static final int MULTIPLE_STREET_RELATIONS = 2604;
61 protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
62 protected static final int OBSOLETE_RELATION = 2606;
63
64 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
65 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
66
67 // CHECKSTYLE.OFF: SingleSpaceSeparator
68 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber";
69 protected static final String ADDR_INTERPOLATION = "addr:interpolation";
70 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
71 protected static final String ADDR_PLACE = "addr:place";
72 protected static final String ADDR_STREET = "addr:street";
73 protected static final String ADDR_SUBURB = "addr:suburb";
74 protected static final String ADDR_CITY = "addr:city";
75 protected static final String ADDR_UNIT = "addr:unit";
76 protected static final String ADDR_FLATS = "addr:flats";
77 protected static final String ADDR_HOUSE_NAME = "addr:housename";
78 protected static final String ADDR_POSTCODE = "addr:postcode";
79 protected static final String ASSOCIATED_STREET = "associatedStreet";
80 protected static final String NAME_TAG = "name";
81 private static final String HOUSE = "house";
82 private static final String STREET = "street";
83 // CHECKSTYLE.ON: SingleSpaceSeparator
84
85 private static final BooleanProperty PREF_INCLUDE_BLDG_POI =
86 new BooleanProperty(ValidatorPrefHelper.PREFIX + "." + OpeningHourTest.class.getSimpleName() + "." + "includebuildingpois", false);
87 private final JCheckBox checkboxIncludeBldgPOI = new JCheckBox(
88 /* I18n: Label text for checkbox choosing to validate addresses for all types of objects, not just plain addresses */
89 tr("Include POIs like amenities, offices, and buildings in duplicate address detection"));
90 private boolean includeBldgAndPOI;
91
92 private Map<String, Collection<OsmPrimitive>> knownAddresses;
93 private Set<String> ignoredAddresses;
94
95 /**
96 * Constructor
97 */
98 public Addresses() {
99 /* I18n: Label text for checkbox choosing to validate addresses */
100 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
101 }
102
103 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
104 final List<Relation> list = p.referrers(Relation.class)
105 .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
106 .collect(Collectors.toList());
107 if (list.size() > 1) {
108 Severity level;
109 // warning level only if several relations have different names, see #10945
110 final String name = list.get(0).get("name");
111 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
112 level = Severity.WARNING;
113 } else {
114 level = Severity.OTHER;
115 }
116 List<OsmPrimitive> errorList = new ArrayList<>(list);
117 errorList.add(0, p);
118 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
119 .message(tr("Multiple associatedStreet relations"))
120 .primitives(errorList)
121 .build());
122 }
123 return list;
124 }
125
126 /**
127 * Checks for house numbers for which the street is unknown.
128 * @param p primitive to test
129 * @return error found, or null
130 */
131 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) {
132 // Find house number without proper location
133 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
134 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)
135 && getAndCheckAssociatedStreets(p).isEmpty()
136 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
137 // no street found
138 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
139 .message(tr("House number without street"))
140 .primitives(p)
141 .build();
142 errors.add(e);
143 return e;
144 }
145 return null;
146 }
147
148 static boolean isPOI(OsmPrimitive p) {
149 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name") ||
150 p.hasTag("barrier", "entrance", "gate");
151 }
152
153 static boolean hasAddress(OsmPrimitive p) {
154 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
155 }
156
157 /**
158 * adds the OsmPrimitive to the address map if it complies to the restrictions
159 * @param p OsmPrimitive that has an address
160 */
161 private void collectAddress(OsmPrimitive p) {
162 if (includeBldgAndPOI || !isPOI(p)) {
163 for (String simplifiedAddress : getSimplifiedAddresses(p)) {
164 if (!ignoredAddresses.contains(simplifiedAddress)) {
165 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
166 }
167 }
168 }
169 }
170
171 protected void initAddressMap(OsmPrimitive primitive) {
172 knownAddresses = new HashMap<>();
173 ignoredAddresses = new HashSet<>();
174 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
175 if ((includeBldgAndPOI || p instanceof Node) && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
176 for (OsmPrimitive r : p.getReferrers()) {
177 if (hasAddress(r)) {
178 // ignore addresses of buildings that are connected to addr:unit nodes
179 // it's quite reasonable that there are more buildings with this address
180 for (String simplifiedAddress : getSimplifiedAddresses(r)) {
181 if (!ignoredAddresses.contains(simplifiedAddress)) {
182 ignoredAddresses.add(simplifiedAddress);
183 } else {
184 knownAddresses.remove(simplifiedAddress);
185 }
186 }
187 }
188 }
189 }
190 if (hasAddress(p)) {
191 collectAddress(p);
192 }
193 }
194 }
195
196 @Override
197 public void startTest(ProgressMonitor progressMonitor) {
198 super.startTest(progressMonitor);
199 this.includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get();
200 }
201
202 @Override
203 public void endTest() {
204 knownAddresses = null;
205 ignoredAddresses = null;
206 super.endTest();
207 }
208
209 protected List<TestError> checkForDuplicate(OsmPrimitive p) {
210 if (knownAddresses == null) {
211 initAddressMap(p);
212 }
213 if ((includeBldgAndPOI || !isPOI(p)) && hasAddress(p)) {
214 List<TestError> result = new ArrayList<>();
215 for (String simplifiedAddress : getSimplifiedAddresses(p)) {
216 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) {
217 double maxDistance = MAX_DUPLICATE_DISTANCE.get();
218 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
219 if (p == p2) {
220 continue;
221 }
222 Severity severityLevel;
223 String city1 = p.get(ADDR_CITY);
224 String city2 = p2.get(ADDR_CITY);
225 String name1 = p.get(NAME_TAG);
226 String name2 = p2.get(NAME_TAG);
227 double distance = getDistance(p, p2);
228 if (city1 != null && city2 != null) {
229 if (city1.equals(city2)) {
230 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE)
231 || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE)))
232 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB)
233 || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) {
234 severityLevel = Severity.WARNING;
235 } else {
236 // address including city identical but postcode or suburb differs
237 // most likely perfectly fine
238 severityLevel = Severity.OTHER;
239 }
240 } else {
241 // address differs only by city - notify if very close, otherwise ignore
242 if (distance < maxDistance) {
243 severityLevel = Severity.OTHER;
244 } else {
245 continue;
246 }
247 }
248 } else {
249 // at least one address has no city specified
250 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE)
251 && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
252 // address including postcode identical
253 severityLevel = Severity.WARNING;
254 } else {
255 // city/postcode unclear - warn if very close, otherwise only notify
256 // TODO: get city from surrounding boundaries?
257 if (distance < maxDistance) {
258 severityLevel = Severity.WARNING;
259 } else {
260 severityLevel = Severity.OTHER;
261 }
262 }
263 }
264 if (severityLevel == Severity.WARNING && !Objects.equals(name1, name2)) {
265 // since multiple objects can exist at one address, a different name tag isn't very concerning
266 severityLevel = Severity.OTHER;
267 }
268 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
269 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
270 .primitives(Arrays.asList(p, p2)).build());
271 }
272 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
273 }
274 }
275 errors.addAll(result);
276 return Collections.unmodifiableList(result);
277 }
278 return Collections.emptyList();
279 }
280
281 static List<String> getSimplifiedAddresses(OsmPrimitive p) {
282 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
283 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
284 return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of(
285 simplifiedStreetName.replaceAll("[ -]", ""),
286 addrHouseNumber,
287 p.get(ADDR_HOUSE_NAME),
288 p.get(ADDR_UNIT),
289 p.get(ADDR_FLATS))
290 .filter(Objects::nonNull)
291 .collect(Collectors.joining(" ")))
292 .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList());
293 }
294
295 /**
296 * Split addr:housenumber on , and ; (common separators)
297 *
298 * @param houseNumber The housenumber to be split
299 * @return A list of addr:housenumber equivalents
300 */
301 static List<String> expandHouseNumber(String houseNumber) {
302 return Arrays.asList(houseNumber.split("[,;]", -1));
303 }
304
305 @Override
306 public void visit(Node n) {
307 checkHouseNumbersWithoutStreet(n);
308 checkForDuplicate(n);
309 }
310
311 @Override
312 public void visit(Way w) {
313 checkHouseNumbersWithoutStreet(w);
314 checkForDuplicate(w);
315 }
316
317 @Override
318 public void visit(Relation r) {
319 checkHouseNumbersWithoutStreet(r);
320 checkForDuplicate(r);
321 if (r.hasTag("type", ASSOCIATED_STREET)) {
322 checkIfObsolete(r);
323 // Used to count occurrences of each house number in order to find duplicates
324 Map<String, List<OsmPrimitive>> map = new HashMap<>();
325 // Used to detect different street names
326 String relationName = r.get("name");
327 Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
328 // Used to check distance
329 Set<OsmPrimitive> houses = new HashSet<>();
330 Set<Way> street = new HashSet<>();
331 for (RelationMember m : r.getMembers()) {
332 String role = m.getRole();
333 OsmPrimitive p = m.getMember();
334 if (HOUSE.equals(role)) {
335 houses.add(p);
336 String number = p.get(ADDR_HOUSE_NUMBER);
337 if (number != null) {
338 number = number.trim().toUpperCase(Locale.ENGLISH);
339 List<OsmPrimitive> list = map.computeIfAbsent(number, k -> new ArrayList<>());
340 list.add(p);
341 }
342 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
343 if (wrongStreetNames.isEmpty()) {
344 wrongStreetNames.add(r);
345 }
346 wrongStreetNames.add(p);
347 }
348 } else if (STREET.equals(role)) {
349 if (p instanceof Way) {
350 street.add((Way) p);
351 }
352 if (relationName != null && p.hasTagDifferent("name", relationName)) {
353 if (wrongStreetNames.isEmpty()) {
354 wrongStreetNames.add(r);
355 }
356 wrongStreetNames.add(p);
357 }
358 }
359 }
360 // Report duplicate house numbers
361 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
362 List<OsmPrimitive> list = entry.getValue();
363 if (list.size() > 1) {
364 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
365 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
366 .primitives(list)
367 .build());
368 }
369 }
370 // Report wrong street names
371 if (!wrongStreetNames.isEmpty()) {
372 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
373 .message(tr("Multiple street names in relation"))
374 .primitives(wrongStreetNames)
375 .build());
376 }
377 // Report addresses too far away
378 if (!street.isEmpty()) {
379 for (OsmPrimitive house : houses) {
380 if (house.isUsable()) {
381 checkDistance(house, street);
382 }
383 }
384 }
385 }
386 }
387
388 /**
389 * returns rough distance between two OsmPrimitives
390 * @param a primitive a
391 * @param b primitive b
392 * @return distance of center of bounding boxes in meters
393 */
394 static double getDistance(OsmPrimitive a, OsmPrimitive b) {
395 if (a instanceof ILatLon && b instanceof ILatLon) {
396 return ((ILatLon) a).greatCircleDistance((ILatLon) b);
397 }
398 ILatLon centerA = a.getBBox().getCenter();
399 ILatLon centerB = b.getBBox().getCenter();
400 return centerA.greatCircleDistance(centerB);
401 }
402
403 protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
404 EastNorth centroid;
405 if (house instanceof Node) {
406 centroid = ((Node) house).getEastNorth();
407 } else if (house instanceof Way) {
408 List<Node> nodes = ((Way) house).getNodes();
409 if (house.hasKey(ADDR_INTERPOLATION)) {
410 for (Node n : nodes) {
411 if (n.hasKey(ADDR_HOUSE_NUMBER)) {
412 checkDistance(n, street);
413 }
414 }
415 return;
416 }
417 centroid = Geometry.getCentroid(nodes);
418 } else {
419 return; // TODO handle multipolygon houses ?
420 }
421 if (centroid == null) return; // fix #8305
422 double maxDistance = MAX_STREET_DISTANCE.get();
423 boolean hasIncompleteWays = false;
424 for (Way streetPart : street) {
425 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
426 EastNorth p1 = chunk.a.getEastNorth();
427 EastNorth p2 = chunk.b.getEastNorth();
428 if (p1 != null && p2 != null) {
429 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
430 if (closest.distance(centroid) <= maxDistance) {
431 return;
432 }
433 } else {
434 Logging.warn("Addresses test skipped chunk "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
435 }
436 }
437 if (!hasIncompleteWays && streetPart.isIncomplete()) {
438 hasIncompleteWays = true;
439 }
440 }
441 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
442 if (hasIncompleteWays) return;
443 List<OsmPrimitive> errorList = new ArrayList<>(street);
444 errorList.add(0, house);
445 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
446 .message(tr("House number too far from street"))
447 .primitives(errorList)
448 .build());
449 }
450
451 /**
452 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
453 * are complete and don't contain any information which isn't also tagged on the members.
454 * The strategy is to avoid any false positive.
455 * @param r the relation
456 */
457 private void checkIfObsolete(Relation r) {
458 if (r.isIncomplete())
459 return;
460 /* array of country codes for which the test should be performed. For now, only Germany */
461 String[] countryCodes = {"DE"};
462 TagMap neededtagsForHouse = new TagMap();
463 for (Entry<String, String> tag : r.getKeys().entrySet()) {
464 String key = tag.getKey();
465 if (key.startsWith("name:")) {
466 return; // maybe check if all members have corresponding tags?
467 } else if (key.startsWith("addr:")) {
468 neededtagsForHouse.put(key, tag.getValue());
469 } else {
470 switch (key) {
471 case "name":
472 case "type":
473 case "source":
474 break;
475 default:
476 // unexpected tag in relation
477 return;
478 }
479 }
480 }
481
482 for (RelationMember m : r.getMembers()) {
483 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes))
484 return;
485
486 String role = m.getRole();
487 if ("".equals(role)) {
488 if (m.isWay() && m.getMember().hasKey("highway")) {
489 role = STREET;
490 } else if (m.getMember().hasTag("building"))
491 role = HOUSE;
492 }
493 switch (role) {
494 case HOUSE:
495 case "addr:houselink":
496 case "address":
497 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
498 return;
499 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
500 if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
501 return;
502 }
503 break;
504 case STREET:
505 if (!m.getMember().hasTag("name") && r.hasTag("name"))
506 return;
507 break;
508 default:
509 // unknown role: don't create auto-fix
510 return;
511 }
512 }
513 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
514 .message(tr("Relation is obsolete"))
515 .primitives(r)
516 .build());
517 }
518
519 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
520 if (countryCodes.length == 0)
521 return true;
522 LatLon center = null;
523
524 if (m.isNode()) {
525 center = m.getNode().getCoor();
526 } else if (m.isWay()) {
527 center = m.getWay().getBBox().getCenter();
528 } else if (m.isRelation() && m.getRelation().isMultipolygon()) {
529 center = m.getRelation().getBBox().getCenter();
530 }
531 if (center == null)
532 return false;
533 for (String country : countryCodes) {
534 if (Territories.isIso3166Code(country, center))
535 return true;
536 }
537 return false;
538 }
539
540 /**
541 * remove obsolete relation.
542 */
543 @Override
544 public Command fixError(TestError testError) {
545 return new DeleteCommand(testError.getPrimitives());
546 }
547
548 @Override
549 public boolean isFixable(TestError testError) {
550 if (!(testError.getTester() instanceof Addresses))
551 return false;
552 return testError.getCode() == OBSOLETE_RELATION;
553 }
554
555 @Override
556 public void addGui(JPanel testPanel) {
557 super.addGui(testPanel);
558 checkboxIncludeBldgPOI.setSelected(PREF_INCLUDE_BLDG_POI.get());
559 testPanel.add(checkboxIncludeBldgPOI, GBC.eol().insets(20, 0, 0, 0));
560 }
561
562 @Override
563 public boolean ok() {
564 super.ok();
565 PREF_INCLUDE_BLDG_POI.put(checkboxIncludeBldgPOI.isSelected());
566 includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get();
567 return false;
568 }
569
570}
Note: See TracBrowser for help on using the repository browser.