001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.util.ArrayList; 008import java.util.Arrays; 009import java.util.Collection; 010import java.util.Collections; 011import java.util.HashMap; 012import java.util.HashSet; 013import java.util.List; 014import java.util.Locale; 015import java.util.Map; 016import java.util.Map.Entry; 017import java.util.Objects; 018import java.util.Set; 019import java.util.stream.Collectors; 020import java.util.stream.Stream; 021 022import javax.swing.JCheckBox; 023import javax.swing.JPanel; 024 025import org.openstreetmap.josm.command.Command; 026import org.openstreetmap.josm.command.DeleteCommand; 027import org.openstreetmap.josm.data.coor.EastNorth; 028import org.openstreetmap.josm.data.coor.ILatLon; 029import org.openstreetmap.josm.data.coor.LatLon; 030import org.openstreetmap.josm.data.osm.Node; 031import org.openstreetmap.josm.data.osm.OsmPrimitive; 032import org.openstreetmap.josm.data.osm.Relation; 033import org.openstreetmap.josm.data.osm.RelationMember; 034import org.openstreetmap.josm.data.osm.TagMap; 035import org.openstreetmap.josm.data.osm.Way; 036import org.openstreetmap.josm.data.preferences.BooleanProperty; 037import org.openstreetmap.josm.data.preferences.DoubleProperty; 038import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 039import org.openstreetmap.josm.data.validation.Severity; 040import org.openstreetmap.josm.data.validation.Test; 041import org.openstreetmap.josm.data.validation.TestError; 042import org.openstreetmap.josm.gui.progress.ProgressMonitor; 043import org.openstreetmap.josm.tools.GBC; 044import org.openstreetmap.josm.tools.Geometry; 045import org.openstreetmap.josm.tools.Logging; 046import org.openstreetmap.josm.tools.Pair; 047import org.openstreetmap.josm.tools.SubclassFilteredCollection; 048import org.openstreetmap.josm.tools.Territories; 049import org.openstreetmap.josm.tools.Utils; 050 051/** 052 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations. 053 * @since 5644 054 */ 055public class Addresses extends Test { 056 057 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601; 058 protected static final int DUPLICATE_HOUSE_NUMBER = 2602; 059 protected static final int MULTIPLE_STREET_NAMES = 2603; 060 protected static final int MULTIPLE_STREET_RELATIONS = 2604; 061 protected static final int HOUSE_NUMBER_TOO_FAR = 2605; 062 protected static final int OBSOLETE_RELATION = 2606; 063 064 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0); 065 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0); 066 067 // CHECKSTYLE.OFF: SingleSpaceSeparator 068 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber"; 069 protected static final String ADDR_INTERPOLATION = "addr:interpolation"; 070 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood"; 071 protected static final String ADDR_PLACE = "addr:place"; 072 protected static final String ADDR_STREET = "addr:street"; 073 protected static final String ADDR_SUBURB = "addr:suburb"; 074 protected static final String ADDR_CITY = "addr:city"; 075 protected static final String ADDR_UNIT = "addr:unit"; 076 protected static final String ADDR_FLATS = "addr:flats"; 077 protected static final String ADDR_HOUSE_NAME = "addr:housename"; 078 protected static final String ADDR_POSTCODE = "addr:postcode"; 079 protected static final String ASSOCIATED_STREET = "associatedStreet"; 080 protected static final String NAME_TAG = "name"; 081 private static final String HOUSE = "house"; 082 private static final String STREET = "street"; 083 // CHECKSTYLE.ON: SingleSpaceSeparator 084 085 private static final BooleanProperty PREF_INCLUDE_BLDG_POI = 086 new BooleanProperty(ValidatorPrefHelper.PREFIX + "." + OpeningHourTest.class.getSimpleName() + "." + "includebuildingpois", false); 087 private final JCheckBox checkboxIncludeBldgPOI = new JCheckBox( 088 /* I18n: Label text for checkbox choosing to validate addresses for all types of objects, not just plain addresses */ 089 tr("Include POIs like amenities, offices, and buildings in duplicate address detection")); 090 private boolean includeBldgAndPOI; 091 092 private Map<String, Collection<OsmPrimitive>> knownAddresses; 093 private Set<String> ignoredAddresses; 094 095 /** 096 * Constructor 097 */ 098 public Addresses() { 099 /* I18n: Label text for checkbox choosing to validate addresses */ 100 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations.")); 101 } 102 103 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) { 104 final List<Relation> list = p.referrers(Relation.class) 105 .filter(r -> r.hasTag("type", ASSOCIATED_STREET)) 106 .collect(Collectors.toList()); 107 if (list.size() > 1) { 108 Severity level; 109 // warning level only if several relations have different names, see #10945 110 final String name = list.get(0).get("name"); 111 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) { 112 level = Severity.WARNING; 113 } else { 114 level = Severity.OTHER; 115 } 116 List<OsmPrimitive> errorList = new ArrayList<>(list); 117 errorList.add(0, p); 118 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS) 119 .message(tr("Multiple associatedStreet relations")) 120 .primitives(errorList) 121 .build()); 122 } 123 return list; 124 } 125 126 /** 127 * Checks for house numbers for which the street is unknown. 128 * @param p primitive to test 129 * @return error found, or null 130 */ 131 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) { 132 // Find house number without proper location 133 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation) 134 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD) 135 && getAndCheckAssociatedStreets(p).isEmpty() 136 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) { 137 // no street found 138 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET) 139 .message(tr("House number without street")) 140 .primitives(p) 141 .build(); 142 errors.add(e); 143 return e; 144 } 145 return null; 146 } 147 148 static boolean isPOI(OsmPrimitive p) { 149 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name") || 150 p.hasTag("barrier", "entrance", "gate"); 151 } 152 153 static boolean hasAddress(OsmPrimitive p) { 154 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE); 155 } 156 157 /** 158 * adds the OsmPrimitive to the address map if it complies to the restrictions 159 * @param p OsmPrimitive that has an address 160 */ 161 private void collectAddress(OsmPrimitive p) { 162 if (includeBldgAndPOI || !isPOI(p)) { 163 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 164 if (!ignoredAddresses.contains(simplifiedAddress)) { 165 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p); 166 } 167 } 168 } 169 } 170 171 protected void initAddressMap(OsmPrimitive primitive) { 172 knownAddresses = new HashMap<>(); 173 ignoredAddresses = new HashSet<>(); 174 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) { 175 if ((includeBldgAndPOI || p instanceof Node) && p.hasKey(ADDR_UNIT, ADDR_FLATS)) { 176 for (OsmPrimitive r : p.getReferrers()) { 177 if (hasAddress(r)) { 178 // ignore addresses of buildings that are connected to addr:unit nodes 179 // it's quite reasonable that there are more buildings with this address 180 for (String simplifiedAddress : getSimplifiedAddresses(r)) { 181 if (!ignoredAddresses.contains(simplifiedAddress)) { 182 ignoredAddresses.add(simplifiedAddress); 183 } else { 184 knownAddresses.remove(simplifiedAddress); 185 } 186 } 187 } 188 } 189 } 190 if (hasAddress(p)) { 191 collectAddress(p); 192 } 193 } 194 } 195 196 @Override 197 public void startTest(ProgressMonitor progressMonitor) { 198 super.startTest(progressMonitor); 199 this.includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get(); 200 } 201 202 @Override 203 public void endTest() { 204 knownAddresses = null; 205 ignoredAddresses = null; 206 super.endTest(); 207 } 208 209 protected List<TestError> checkForDuplicate(OsmPrimitive p) { 210 if (knownAddresses == null) { 211 initAddressMap(p); 212 } 213 if ((includeBldgAndPOI || !isPOI(p)) && hasAddress(p)) { 214 List<TestError> result = new ArrayList<>(); 215 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 216 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) { 217 double maxDistance = MAX_DUPLICATE_DISTANCE.get(); 218 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) { 219 if (p == p2) { 220 continue; 221 } 222 Severity severityLevel; 223 String city1 = p.get(ADDR_CITY); 224 String city2 = p2.get(ADDR_CITY); 225 String name1 = p.get(NAME_TAG); 226 String name2 = p2.get(NAME_TAG); 227 double distance = getDistance(p, p2); 228 if (city1 != null && city2 != null) { 229 if (city1.equals(city2)) { 230 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) 231 || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) 232 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB) 233 || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) { 234 severityLevel = Severity.WARNING; 235 } else { 236 // address including city identical but postcode or suburb differs 237 // most likely perfectly fine 238 severityLevel = Severity.OTHER; 239 } 240 } else { 241 // address differs only by city - notify if very close, otherwise ignore 242 if (distance < maxDistance) { 243 severityLevel = Severity.OTHER; 244 } else { 245 continue; 246 } 247 } 248 } else { 249 // at least one address has no city specified 250 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) 251 && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 252 // address including postcode identical 253 severityLevel = Severity.WARNING; 254 } else { 255 // city/postcode unclear - warn if very close, otherwise only notify 256 // TODO: get city from surrounding boundaries? 257 if (distance < maxDistance) { 258 severityLevel = Severity.WARNING; 259 } else { 260 severityLevel = Severity.OTHER; 261 } 262 } 263 } 264 if (severityLevel == Severity.WARNING && !Objects.equals(name1, name2)) { 265 // since multiple objects can exist at one address, a different name tag isn't very concerning 266 severityLevel = Severity.OTHER; 267 } 268 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER) 269 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance) 270 .primitives(Arrays.asList(p, p2)).build()); 271 } 272 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times 273 } 274 } 275 errors.addAll(result); 276 return Collections.unmodifiableList(result); 277 } 278 return Collections.emptyList(); 279 } 280 281 static List<String> getSimplifiedAddresses(OsmPrimitive p) { 282 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE); 283 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal 284 return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of( 285 simplifiedStreetName.replaceAll("[ -]", ""), 286 addrHouseNumber, 287 p.get(ADDR_HOUSE_NAME), 288 p.get(ADDR_UNIT), 289 p.get(ADDR_FLATS)) 290 .filter(Objects::nonNull) 291 .collect(Collectors.joining(" "))) 292 .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList()); 293 } 294 295 /** 296 * Split addr:housenumber on , and ; (common separators) 297 * 298 * @param houseNumber The housenumber to be split 299 * @return A list of addr:housenumber equivalents 300 */ 301 static List<String> expandHouseNumber(String houseNumber) { 302 return Arrays.asList(houseNumber.split("[,;]", -1)); 303 } 304 305 @Override 306 public void visit(Node n) { 307 checkHouseNumbersWithoutStreet(n); 308 checkForDuplicate(n); 309 } 310 311 @Override 312 public void visit(Way w) { 313 checkHouseNumbersWithoutStreet(w); 314 checkForDuplicate(w); 315 } 316 317 @Override 318 public void visit(Relation r) { 319 checkHouseNumbersWithoutStreet(r); 320 checkForDuplicate(r); 321 if (r.hasTag("type", ASSOCIATED_STREET)) { 322 checkIfObsolete(r); 323 // Used to count occurrences of each house number in order to find duplicates 324 Map<String, List<OsmPrimitive>> map = new HashMap<>(); 325 // Used to detect different street names 326 String relationName = r.get("name"); 327 Set<OsmPrimitive> wrongStreetNames = new HashSet<>(); 328 // Used to check distance 329 Set<OsmPrimitive> houses = new HashSet<>(); 330 Set<Way> street = new HashSet<>(); 331 for (RelationMember m : r.getMembers()) { 332 String role = m.getRole(); 333 OsmPrimitive p = m.getMember(); 334 if (HOUSE.equals(role)) { 335 houses.add(p); 336 String number = p.get(ADDR_HOUSE_NUMBER); 337 if (number != null) { 338 number = number.trim().toUpperCase(Locale.ENGLISH); 339 List<OsmPrimitive> list = map.computeIfAbsent(number, k -> new ArrayList<>()); 340 list.add(p); 341 } 342 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) { 343 if (wrongStreetNames.isEmpty()) { 344 wrongStreetNames.add(r); 345 } 346 wrongStreetNames.add(p); 347 } 348 } else if (STREET.equals(role)) { 349 if (p instanceof Way) { 350 street.add((Way) p); 351 } 352 if (relationName != null && p.hasTagDifferent("name", relationName)) { 353 if (wrongStreetNames.isEmpty()) { 354 wrongStreetNames.add(r); 355 } 356 wrongStreetNames.add(p); 357 } 358 } 359 } 360 // Report duplicate house numbers 361 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) { 362 List<OsmPrimitive> list = entry.getValue(); 363 if (list.size() > 1) { 364 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER) 365 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey()) 366 .primitives(list) 367 .build()); 368 } 369 } 370 // Report wrong street names 371 if (!wrongStreetNames.isEmpty()) { 372 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES) 373 .message(tr("Multiple street names in relation")) 374 .primitives(wrongStreetNames) 375 .build()); 376 } 377 // Report addresses too far away 378 if (!street.isEmpty()) { 379 for (OsmPrimitive house : houses) { 380 if (house.isUsable()) { 381 checkDistance(house, street); 382 } 383 } 384 } 385 } 386 } 387 388 /** 389 * returns rough distance between two OsmPrimitives 390 * @param a primitive a 391 * @param b primitive b 392 * @return distance of center of bounding boxes in meters 393 */ 394 static double getDistance(OsmPrimitive a, OsmPrimitive b) { 395 if (a instanceof ILatLon && b instanceof ILatLon) { 396 return ((ILatLon) a).greatCircleDistance((ILatLon) b); 397 } 398 ILatLon centerA = a.getBBox().getCenter(); 399 ILatLon centerB = b.getBBox().getCenter(); 400 return centerA.greatCircleDistance(centerB); 401 } 402 403 protected void checkDistance(OsmPrimitive house, Collection<Way> street) { 404 EastNorth centroid; 405 if (house instanceof Node) { 406 centroid = ((Node) house).getEastNorth(); 407 } else if (house instanceof Way) { 408 List<Node> nodes = ((Way) house).getNodes(); 409 if (house.hasKey(ADDR_INTERPOLATION)) { 410 for (Node n : nodes) { 411 if (n.hasKey(ADDR_HOUSE_NUMBER)) { 412 checkDistance(n, street); 413 } 414 } 415 return; 416 } 417 centroid = Geometry.getCentroid(nodes); 418 } else { 419 return; // TODO handle multipolygon houses ? 420 } 421 if (centroid == null) return; // fix #8305 422 double maxDistance = MAX_STREET_DISTANCE.get(); 423 boolean hasIncompleteWays = false; 424 for (Way streetPart : street) { 425 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) { 426 EastNorth p1 = chunk.a.getEastNorth(); 427 EastNorth p2 = chunk.b.getEastNorth(); 428 if (p1 != null && p2 != null) { 429 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid); 430 if (closest.distance(centroid) <= maxDistance) { 431 return; 432 } 433 } else { 434 Logging.warn("Addresses test skipped chunk "+chunk+" for street part "+streetPart+" because p1 or p2 is null"); 435 } 436 } 437 if (!hasIncompleteWays && streetPart.isIncomplete()) { 438 hasIncompleteWays = true; 439 } 440 } 441 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314) 442 if (hasIncompleteWays) return; 443 List<OsmPrimitive> errorList = new ArrayList<>(street); 444 errorList.add(0, house); 445 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR) 446 .message(tr("House number too far from street")) 447 .primitives(errorList) 448 .build()); 449 } 450 451 /** 452 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which 453 * are complete and don't contain any information which isn't also tagged on the members. 454 * The strategy is to avoid any false positive. 455 * @param r the relation 456 */ 457 private void checkIfObsolete(Relation r) { 458 if (r.isIncomplete()) 459 return; 460 /* array of country codes for which the test should be performed. For now, only Germany */ 461 String[] countryCodes = {"DE"}; 462 TagMap neededtagsForHouse = new TagMap(); 463 for (Entry<String, String> tag : r.getKeys().entrySet()) { 464 String key = tag.getKey(); 465 if (key.startsWith("name:")) { 466 return; // maybe check if all members have corresponding tags? 467 } else if (key.startsWith("addr:")) { 468 neededtagsForHouse.put(key, tag.getValue()); 469 } else { 470 switch (key) { 471 case "name": 472 case "type": 473 case "source": 474 break; 475 default: 476 // unexpected tag in relation 477 return; 478 } 479 } 480 } 481 482 for (RelationMember m : r.getMembers()) { 483 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes)) 484 return; 485 486 String role = m.getRole(); 487 if ("".equals(role)) { 488 if (m.isWay() && m.getMember().hasKey("highway")) { 489 role = STREET; 490 } else if (m.getMember().hasTag("building")) 491 role = HOUSE; 492 } 493 switch (role) { 494 case HOUSE: 495 case "addr:houselink": 496 case "address": 497 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER)) 498 return; 499 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) { 500 if (!m.getMember().hasTag(tag.getKey(), tag.getValue())) 501 return; 502 } 503 break; 504 case STREET: 505 if (!m.getMember().hasTag("name") && r.hasTag("name")) 506 return; 507 break; 508 default: 509 // unknown role: don't create auto-fix 510 return; 511 } 512 } 513 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION) 514 .message(tr("Relation is obsolete")) 515 .primitives(r) 516 .build()); 517 } 518 519 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) { 520 if (countryCodes.length == 0) 521 return true; 522 LatLon center = null; 523 524 if (m.isNode()) { 525 center = m.getNode().getCoor(); 526 } else if (m.isWay()) { 527 center = m.getWay().getBBox().getCenter(); 528 } else if (m.isRelation() && m.getRelation().isMultipolygon()) { 529 center = m.getRelation().getBBox().getCenter(); 530 } 531 if (center == null) 532 return false; 533 for (String country : countryCodes) { 534 if (Territories.isIso3166Code(country, center)) 535 return true; 536 } 537 return false; 538 } 539 540 /** 541 * remove obsolete relation. 542 */ 543 @Override 544 public Command fixError(TestError testError) { 545 return new DeleteCommand(testError.getPrimitives()); 546 } 547 548 @Override 549 public boolean isFixable(TestError testError) { 550 if (!(testError.getTester() instanceof Addresses)) 551 return false; 552 return testError.getCode() == OBSOLETE_RELATION; 553 } 554 555 @Override 556 public void addGui(JPanel testPanel) { 557 super.addGui(testPanel); 558 checkboxIncludeBldgPOI.setSelected(PREF_INCLUDE_BLDG_POI.get()); 559 testPanel.add(checkboxIncludeBldgPOI, GBC.eol().insets(20, 0, 0, 0)); 560 } 561 562 @Override 563 public boolean ok() { 564 super.ok(); 565 PREF_INCLUDE_BLDG_POI.put(checkboxIncludeBldgPOI.isSelected()); 566 includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get(); 567 return false; 568 } 569 570}