source: josm/trunk/src/org/openstreetmap/josm/data/validation/tests/Addresses.java@ 17787

Last change on this file since 17787 was 17787, checked in by simon04, 5 years ago

fix #20741 - Various code simplifications (patch by gaben)
  • Property svn:eol-style set to native
File size: 22.1 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.data.validation.tests;
3
4import static org.openstreetmap.josm.tools.I18n.marktr;
5import static org.openstreetmap.josm.tools.I18n.tr;
6
7import java.util.ArrayList;
8import java.util.Arrays;
9import java.util.Collection;
10import java.util.Collections;
11import java.util.HashMap;
12import java.util.HashSet;
13import java.util.List;
14import java.util.Locale;
15import java.util.Map;
16import java.util.Map.Entry;
17import java.util.Objects;
18import java.util.Set;
19import java.util.stream.Collectors;
20import java.util.stream.Stream;
21
22import org.openstreetmap.josm.command.Command;
23import org.openstreetmap.josm.command.DeleteCommand;
24import org.openstreetmap.josm.data.coor.EastNorth;
25import org.openstreetmap.josm.data.coor.LatLon;
26import org.openstreetmap.josm.data.osm.Node;
27import org.openstreetmap.josm.data.osm.OsmPrimitive;
28import org.openstreetmap.josm.data.osm.Relation;
29import org.openstreetmap.josm.data.osm.RelationMember;
30import org.openstreetmap.josm.data.osm.TagMap;
31import org.openstreetmap.josm.data.osm.Way;
32import org.openstreetmap.josm.data.preferences.DoubleProperty;
33import org.openstreetmap.josm.data.validation.Severity;
34import org.openstreetmap.josm.data.validation.Test;
35import org.openstreetmap.josm.data.validation.TestError;
36import org.openstreetmap.josm.tools.Geometry;
37import org.openstreetmap.josm.tools.Logging;
38import org.openstreetmap.josm.tools.Pair;
39import org.openstreetmap.josm.tools.SubclassFilteredCollection;
40import org.openstreetmap.josm.tools.Territories;
41import org.openstreetmap.josm.tools.Utils;
42
43/**
44 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
45 * @since 5644
46 */
47public class Addresses extends Test {
48
49 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
50 protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
51 protected static final int MULTIPLE_STREET_NAMES = 2603;
52 protected static final int MULTIPLE_STREET_RELATIONS = 2604;
53 protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
54 protected static final int OBSOLETE_RELATION = 2606;
55
56 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
57 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
58
59 // CHECKSTYLE.OFF: SingleSpaceSeparator
60 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber";
61 protected static final String ADDR_INTERPOLATION = "addr:interpolation";
62 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
63 protected static final String ADDR_PLACE = "addr:place";
64 protected static final String ADDR_STREET = "addr:street";
65 protected static final String ADDR_SUBURB = "addr:suburb";
66 protected static final String ADDR_CITY = "addr:city";
67 protected static final String ADDR_UNIT = "addr:unit";
68 protected static final String ADDR_FLATS = "addr:flats";
69 protected static final String ADDR_HOUSE_NAME = "addr:housename";
70 protected static final String ADDR_POSTCODE = "addr:postcode";
71 protected static final String ASSOCIATED_STREET = "associatedStreet";
72 // CHECKSTYLE.ON: SingleSpaceSeparator
73
74 private Map<String, Collection<OsmPrimitive>> knownAddresses;
75 private Set<String> ignoredAddresses;
76
77 /**
78 * Constructor
79 */
80 public Addresses() {
81 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
82 }
83
84 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
85 final List<Relation> list = p.referrers(Relation.class)
86 .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
87 .collect(Collectors.toList());
88 if (list.size() > 1) {
89 Severity level;
90 // warning level only if several relations have different names, see #10945
91 final String name = list.get(0).get("name");
92 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
93 level = Severity.WARNING;
94 } else {
95 level = Severity.OTHER;
96 }
97 List<OsmPrimitive> errorList = new ArrayList<>(list);
98 errorList.add(0, p);
99 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
100 .message(tr("Multiple associatedStreet relations"))
101 .primitives(errorList)
102 .build());
103 }
104 return list;
105 }
106
107 /**
108 * Checks for house numbers for which the street is unknown.
109 * @param p primitive to test
110 * @return error found, or null
111 */
112 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) {
113 // Find house number without proper location
114 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
115 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)
116 && getAndCheckAssociatedStreets(p).isEmpty()
117 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
118 // no street found
119 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
120 .message(tr("House number without street"))
121 .primitives(p)
122 .build();
123 errors.add(e);
124 return e;
125 }
126 return null;
127 }
128
129 static boolean isPOI(OsmPrimitive p) {
130 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name") ||
131 p.hasTag("barrier", "entrance", "gate");
132 }
133
134 static boolean hasAddress(OsmPrimitive p) {
135 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
136 }
137
138 /**
139 * adds the OsmPrimitive to the address map if it complies to the restrictions
140 * @param p OsmPrimitive that has an address
141 */
142 private void collectAddress(OsmPrimitive p) {
143 if (!isPOI(p)) {
144 for (String simplifiedAddress : getSimplifiedAddresses(p)) {
145 if (!ignoredAddresses.contains(simplifiedAddress)) {
146 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
147 }
148 }
149 }
150 }
151
152 protected void initAddressMap(OsmPrimitive primitive) {
153 knownAddresses = new HashMap<>();
154 ignoredAddresses = new HashSet<>();
155 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
156 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
157 for (OsmPrimitive r : p.getReferrers()) {
158 if (hasAddress(r)) {
159 // ignore addresses of buildings that are connected to addr:unit nodes
160 // it's quite reasonable that there are more buildings with this address
161 for (String simplifiedAddress : getSimplifiedAddresses(r)) {
162 if (!ignoredAddresses.contains(simplifiedAddress)) {
163 ignoredAddresses.add(simplifiedAddress);
164 } else {
165 knownAddresses.remove(simplifiedAddress);
166 }
167 }
168 }
169 }
170 }
171 if (hasAddress(p)) {
172 collectAddress(p);
173 }
174 }
175 }
176
177 @Override
178 public void endTest() {
179 knownAddresses = null;
180 ignoredAddresses = null;
181 super.endTest();
182 }
183
184 protected List<TestError> checkForDuplicate(OsmPrimitive p) {
185 if (knownAddresses == null) {
186 initAddressMap(p);
187 }
188 if (!isPOI(p) && hasAddress(p)) {
189 List<TestError> result = new ArrayList<>();
190 for (String simplifiedAddress : getSimplifiedAddresses(p)) {
191 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) {
192 double maxDistance = MAX_DUPLICATE_DISTANCE.get();
193 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
194 if (p == p2) {
195 continue;
196 }
197 Severity severityLevel;
198 String city1 = p.get(ADDR_CITY);
199 String city2 = p2.get(ADDR_CITY);
200 double distance = getDistance(p, p2);
201 if (city1 != null && city2 != null) {
202 if (city1.equals(city2)) {
203 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE)
204 || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE)))
205 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB)
206 || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) {
207 severityLevel = Severity.WARNING;
208 } else {
209 // address including city identical but postcode or suburb differs
210 // most likely perfectly fine
211 severityLevel = Severity.OTHER;
212 }
213 } else {
214 // address differs only by city - notify if very close, otherwise ignore
215 if (distance < maxDistance) {
216 severityLevel = Severity.OTHER;
217 } else {
218 continue;
219 }
220 }
221 } else {
222 // at least one address has no city specified
223 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE)
224 && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
225 // address including postcode identical
226 severityLevel = Severity.WARNING;
227 } else {
228 // city/postcode unclear - warn if very close, otherwise only notify
229 // TODO: get city from surrounding boundaries?
230 if (distance < maxDistance) {
231 severityLevel = Severity.WARNING;
232 } else {
233 severityLevel = Severity.OTHER;
234 }
235 }
236 }
237 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
238 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
239 .primitives(Arrays.asList(p, p2)).build());
240 }
241 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
242 }
243 }
244 errors.addAll(result);
245 return Collections.unmodifiableList(result);
246 }
247 return Collections.emptyList();
248 }
249
250 static List<String> getSimplifiedAddresses(OsmPrimitive p) {
251 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
252 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
253 return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of(
254 simplifiedStreetName.replaceAll("[ -]", ""),
255 addrHouseNumber,
256 p.get(ADDR_HOUSE_NAME),
257 p.get(ADDR_UNIT),
258 p.get(ADDR_FLATS))
259 .filter(Objects::nonNull)
260 .collect(Collectors.joining(" ")))
261 .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList());
262 }
263
264 /**
265 * Split addr:housenumber on , and ; (common separators)
266 *
267 * @param houseNumber The housenumber to be split
268 * @return A list of addr:housenumber equivalents
269 */
270 static List<String> expandHouseNumber(String houseNumber) {
271 return Arrays.asList(houseNumber.split("[,;]", -1));
272 }
273
274 @Override
275 public void visit(Node n) {
276 checkHouseNumbersWithoutStreet(n);
277 checkForDuplicate(n);
278 }
279
280 @Override
281 public void visit(Way w) {
282 checkHouseNumbersWithoutStreet(w);
283 checkForDuplicate(w);
284 }
285
286 @Override
287 public void visit(Relation r) {
288 checkHouseNumbersWithoutStreet(r);
289 checkForDuplicate(r);
290 if (r.hasTag("type", ASSOCIATED_STREET)) {
291 checkIfObsolete(r);
292 // Used to count occurrences of each house number in order to find duplicates
293 Map<String, List<OsmPrimitive>> map = new HashMap<>();
294 // Used to detect different street names
295 String relationName = r.get("name");
296 Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
297 // Used to check distance
298 Set<OsmPrimitive> houses = new HashSet<>();
299 Set<Way> street = new HashSet<>();
300 for (RelationMember m : r.getMembers()) {
301 String role = m.getRole();
302 OsmPrimitive p = m.getMember();
303 if ("house".equals(role)) {
304 houses.add(p);
305 String number = p.get(ADDR_HOUSE_NUMBER);
306 if (number != null) {
307 number = number.trim().toUpperCase(Locale.ENGLISH);
308 List<OsmPrimitive> list = map.computeIfAbsent(number, k -> new ArrayList<>());
309 list.add(p);
310 }
311 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
312 if (wrongStreetNames.isEmpty()) {
313 wrongStreetNames.add(r);
314 }
315 wrongStreetNames.add(p);
316 }
317 } else if ("street".equals(role)) {
318 if (p instanceof Way) {
319 street.add((Way) p);
320 }
321 if (relationName != null && p.hasTagDifferent("name", relationName)) {
322 if (wrongStreetNames.isEmpty()) {
323 wrongStreetNames.add(r);
324 }
325 wrongStreetNames.add(p);
326 }
327 }
328 }
329 // Report duplicate house numbers
330 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
331 List<OsmPrimitive> list = entry.getValue();
332 if (list.size() > 1) {
333 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
334 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
335 .primitives(list)
336 .build());
337 }
338 }
339 // Report wrong street names
340 if (!wrongStreetNames.isEmpty()) {
341 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
342 .message(tr("Multiple street names in relation"))
343 .primitives(wrongStreetNames)
344 .build());
345 }
346 // Report addresses too far away
347 if (!street.isEmpty()) {
348 for (OsmPrimitive house : houses) {
349 if (house.isUsable()) {
350 checkDistance(house, street);
351 }
352 }
353 }
354 }
355 }
356
357 /**
358 * returns rough distance between two OsmPrimitives
359 * @param a primitive a
360 * @param b primitive b
361 * @return distance of center of bounding boxes in meters
362 */
363 static double getDistance(OsmPrimitive a, OsmPrimitive b) {
364 LatLon centerA = a.getBBox().getCenter();
365 LatLon centerB = b.getBBox().getCenter();
366 return (centerA.greatCircleDistance(centerB));
367 }
368
369 protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
370 EastNorth centroid;
371 if (house instanceof Node) {
372 centroid = ((Node) house).getEastNorth();
373 } else if (house instanceof Way) {
374 List<Node> nodes = ((Way) house).getNodes();
375 if (house.hasKey(ADDR_INTERPOLATION)) {
376 for (Node n : nodes) {
377 if (n.hasKey(ADDR_HOUSE_NUMBER)) {
378 checkDistance(n, street);
379 }
380 }
381 return;
382 }
383 centroid = Geometry.getCentroid(nodes);
384 } else {
385 return; // TODO handle multipolygon houses ?
386 }
387 if (centroid == null) return; // fix #8305
388 double maxDistance = MAX_STREET_DISTANCE.get();
389 boolean hasIncompleteWays = false;
390 for (Way streetPart : street) {
391 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
392 EastNorth p1 = chunk.a.getEastNorth();
393 EastNorth p2 = chunk.b.getEastNorth();
394 if (p1 != null && p2 != null) {
395 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
396 if (closest.distance(centroid) <= maxDistance) {
397 return;
398 }
399 } else {
400 Logging.warn("Addresses test skipped chunk "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
401 }
402 }
403 if (!hasIncompleteWays && streetPart.isIncomplete()) {
404 hasIncompleteWays = true;
405 }
406 }
407 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
408 if (hasIncompleteWays) return;
409 List<OsmPrimitive> errorList = new ArrayList<>(street);
410 errorList.add(0, house);
411 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
412 .message(tr("House number too far from street"))
413 .primitives(errorList)
414 .build());
415 }
416
417 /**
418 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
419 * are complete and don't contain any information which isn't also tagged on the members.
420 * The strategy is to avoid any false positive.
421 * @param r the relation
422 */
423 private void checkIfObsolete(Relation r) {
424 if (r.isIncomplete())
425 return;
426 /** array of country codes for which the test should be performed. For now, only Germany */
427 String[] countryCodes = {"DE"};
428 TagMap neededtagsForHouse = new TagMap();
429 for (Entry<String, String> tag : r.getKeys().entrySet()) {
430 String key = tag.getKey();
431 if (key.startsWith("name:")) {
432 return; // maybe check if all members have corresponding tags?
433 } else if (key.startsWith("addr:")) {
434 neededtagsForHouse.put(key, tag.getValue());
435 } else {
436 switch (key) {
437 case "name":
438 case "type":
439 case "source":
440 break;
441 default:
442 // unexpected tag in relation
443 return;
444 }
445 }
446 }
447
448 for (RelationMember m : r.getMembers()) {
449 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes))
450 return;
451
452 String role = m.getRole();
453 if ("".equals(role)) {
454 if (m.isWay() && m.getMember().hasKey("highway")) {
455 role = "street";
456 } else if (m.getMember().hasTag("building"))
457 role = "house";
458 }
459 switch (role) {
460 case "house":
461 case "addr:houselink":
462 case "address":
463 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
464 return;
465 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
466 if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
467 return;
468 }
469 break;
470 case "street":
471 if (!m.getMember().hasTag("name") && r.hasTag("name"))
472 return;
473 break;
474 default:
475 // unknown role: don't create auto-fix
476 return;
477 }
478 }
479 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
480 .message(tr("Relation is obsolete"))
481 .primitives(r)
482 .build());
483 }
484
485 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
486 if (countryCodes.length == 0)
487 return true;
488 LatLon center = null;
489
490 if (m.isNode()) {
491 center = m.getNode().getCoor();
492 } else if (m.isWay()) {
493 center = m.getWay().getBBox().getCenter();
494 } else if (m.isRelation() && m.getRelation().isMultipolygon()) {
495 center = m.getRelation().getBBox().getCenter();
496 }
497 if (center == null)
498 return false;
499 for (String country : countryCodes) {
500 if (Territories.isIso3166Code(country, center))
501 return true;
502 }
503 return false;
504 }
505
506 /**
507 * remove obsolete relation.
508 */
509 @Override
510 public Command fixError(TestError testError) {
511 return new DeleteCommand(testError.getPrimitives());
512 }
513
514 @Override
515 public boolean isFixable(TestError testError) {
516 if (!(testError.getTester() instanceof Addresses))
517 return false;
518 return testError.getCode() == OBSOLETE_RELATION;
519 }
520
521}
Note: See TracBrowser for help on using the repository browser.