001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.util.ArrayList;
008import java.util.Arrays;
009import java.util.Collection;
010import java.util.Collections;
011import java.util.HashMap;
012import java.util.HashSet;
013import java.util.List;
014import java.util.Locale;
015import java.util.Map;
016import java.util.Map.Entry;
017import java.util.Objects;
018import java.util.Set;
019import java.util.stream.Collectors;
020import java.util.stream.Stream;
021
022import javax.swing.JCheckBox;
023import javax.swing.JPanel;
024
025import org.openstreetmap.josm.command.Command;
026import org.openstreetmap.josm.command.DeleteCommand;
027import org.openstreetmap.josm.data.coor.EastNorth;
028import org.openstreetmap.josm.data.coor.ILatLon;
029import org.openstreetmap.josm.data.coor.LatLon;
030import org.openstreetmap.josm.data.osm.Node;
031import org.openstreetmap.josm.data.osm.OsmPrimitive;
032import org.openstreetmap.josm.data.osm.Relation;
033import org.openstreetmap.josm.data.osm.RelationMember;
034import org.openstreetmap.josm.data.osm.TagMap;
035import org.openstreetmap.josm.data.osm.Way;
036import org.openstreetmap.josm.data.preferences.BooleanProperty;
037import org.openstreetmap.josm.data.preferences.DoubleProperty;
038import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
039import org.openstreetmap.josm.data.validation.Severity;
040import org.openstreetmap.josm.data.validation.Test;
041import org.openstreetmap.josm.data.validation.TestError;
042import org.openstreetmap.josm.gui.progress.ProgressMonitor;
043import org.openstreetmap.josm.tools.GBC;
044import org.openstreetmap.josm.tools.Geometry;
045import org.openstreetmap.josm.tools.Logging;
046import org.openstreetmap.josm.tools.Pair;
047import org.openstreetmap.josm.tools.SubclassFilteredCollection;
048import org.openstreetmap.josm.tools.Territories;
049import org.openstreetmap.josm.tools.Utils;
050
051/**
052 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
053 * @since 5644
054 */
055public class Addresses extends Test {
056
057    protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
058    protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
059    protected static final int MULTIPLE_STREET_NAMES = 2603;
060    protected static final int MULTIPLE_STREET_RELATIONS = 2604;
061    protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
062    protected static final int OBSOLETE_RELATION = 2606;
063
064    protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
065    protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
066
067    // CHECKSTYLE.OFF: SingleSpaceSeparator
068    protected static final String ADDR_HOUSE_NUMBER  = "addr:housenumber";
069    protected static final String ADDR_INTERPOLATION = "addr:interpolation";
070    protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
071    protected static final String ADDR_PLACE         = "addr:place";
072    protected static final String ADDR_STREET        = "addr:street";
073    protected static final String ADDR_SUBURB        = "addr:suburb";
074    protected static final String ADDR_CITY          = "addr:city";
075    protected static final String ADDR_UNIT          = "addr:unit";
076    protected static final String ADDR_FLATS         = "addr:flats";
077    protected static final String ADDR_HOUSE_NAME    = "addr:housename";
078    protected static final String ADDR_POSTCODE      = "addr:postcode";
079    protected static final String ASSOCIATED_STREET  = "associatedStreet";
080    protected static final String NAME_TAG           = "name";
081    private static final String HOUSE                = "house";
082    private static final String STREET               = "street";
083    // CHECKSTYLE.ON: SingleSpaceSeparator
084
085    private static final BooleanProperty PREF_INCLUDE_BLDG_POI =
086            new BooleanProperty(ValidatorPrefHelper.PREFIX + "." + OpeningHourTest.class.getSimpleName() + "." + "includebuildingpois", false);
087    private final JCheckBox checkboxIncludeBldgPOI = new JCheckBox(
088            /* I18n: Label text for checkbox choosing to validate addresses for all types of objects, not just plain addresses */
089            tr("Include POIs like amenities, offices, and buildings in duplicate address detection"));
090    private boolean includeBldgAndPOI;
091
092    private Map<String, Collection<OsmPrimitive>> knownAddresses;
093    private Set<String> ignoredAddresses;
094
095    /**
096     * Constructor
097     */
098    public Addresses() {
099        /* I18n: Label text for checkbox choosing to validate addresses */
100        super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
101    }
102
103    protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
104        final List<Relation> list = p.referrers(Relation.class)
105                .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
106                .collect(Collectors.toList());
107        if (list.size() > 1) {
108            Severity level;
109            // warning level only if several relations have different names, see #10945
110            final String name = list.get(0).get("name");
111            if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
112                level = Severity.WARNING;
113            } else {
114                level = Severity.OTHER;
115            }
116            List<OsmPrimitive> errorList = new ArrayList<>(list);
117            errorList.add(0, p);
118            errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
119                    .message(tr("Multiple associatedStreet relations"))
120                    .primitives(errorList)
121                    .build());
122        }
123        return list;
124    }
125
126    /**
127     * Checks for house numbers for which the street is unknown.
128     * @param p primitive to test
129     * @return error found, or null
130     */
131    protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) {
132        // Find house number without proper location
133        // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
134        if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)
135            && getAndCheckAssociatedStreets(p).isEmpty()
136            && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
137            // no street found
138            TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
139                .message(tr("House number without street"))
140                .primitives(p)
141                .build();
142            errors.add(e);
143            return e;
144        }
145        return null;
146    }
147
148    static boolean isPOI(OsmPrimitive p) {
149        return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name") ||
150               p.hasTag("barrier", "entrance", "gate");
151    }
152
153    static boolean hasAddress(OsmPrimitive p) {
154        return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
155    }
156
157    /**
158     * adds the OsmPrimitive to the address map if it complies to the restrictions
159     * @param p OsmPrimitive that has an address
160     */
161    private void collectAddress(OsmPrimitive p) {
162        if (includeBldgAndPOI || !isPOI(p)) {
163            for (String simplifiedAddress : getSimplifiedAddresses(p)) {
164                if (!ignoredAddresses.contains(simplifiedAddress)) {
165                    knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
166                }
167            }
168        }
169    }
170
171    protected void initAddressMap(OsmPrimitive primitive) {
172        knownAddresses = new HashMap<>();
173        ignoredAddresses = new HashSet<>();
174        for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
175            if ((includeBldgAndPOI || p instanceof Node) && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
176                for (OsmPrimitive r : p.getReferrers()) {
177                    if (hasAddress(r)) {
178                        // ignore addresses of buildings that are connected to addr:unit nodes
179                        // it's quite reasonable that there are more buildings with this address
180                        for (String simplifiedAddress : getSimplifiedAddresses(r)) {
181                            if (!ignoredAddresses.contains(simplifiedAddress)) {
182                                ignoredAddresses.add(simplifiedAddress);
183                            } else {
184                                knownAddresses.remove(simplifiedAddress);
185                            }
186                        }
187                    }
188                }
189            }
190            if (hasAddress(p)) {
191                collectAddress(p);
192            }
193        }
194    }
195
196    @Override
197    public void startTest(ProgressMonitor progressMonitor) {
198        super.startTest(progressMonitor);
199        this.includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get();
200    }
201
202    @Override
203    public void endTest() {
204        knownAddresses = null;
205        ignoredAddresses = null;
206        super.endTest();
207    }
208
209    protected List<TestError> checkForDuplicate(OsmPrimitive p) {
210        if (knownAddresses == null) {
211            initAddressMap(p);
212        }
213        if ((includeBldgAndPOI || !isPOI(p)) && hasAddress(p)) {
214            List<TestError> result = new ArrayList<>();
215            for (String simplifiedAddress : getSimplifiedAddresses(p)) {
216                if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) {
217                    double maxDistance = MAX_DUPLICATE_DISTANCE.get();
218                    for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
219                        if (p == p2) {
220                            continue;
221                        }
222                        Severity severityLevel;
223                        String city1 = p.get(ADDR_CITY);
224                        String city2 = p2.get(ADDR_CITY);
225                        String name1 = p.get(NAME_TAG);
226                        String name2 = p2.get(NAME_TAG);
227                        double distance = getDistance(p, p2);
228                        if (city1 != null && city2 != null) {
229                            if (city1.equals(city2)) {
230                                if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE)
231                                        || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE)))
232                                        && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB)
233                                                || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) {
234                                    severityLevel = Severity.WARNING;
235                                } else {
236                                    // address including city identical but postcode or suburb differs
237                                    // most likely perfectly fine
238                                    severityLevel = Severity.OTHER;
239                                }
240                            } else {
241                                // address differs only by city - notify if very close, otherwise ignore
242                                if (distance < maxDistance) {
243                                    severityLevel = Severity.OTHER;
244                                } else {
245                                    continue;
246                                }
247                            }
248                        } else {
249                            // at least one address has no city specified
250                            if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE)
251                                    && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
252                                // address including postcode identical
253                                severityLevel = Severity.WARNING;
254                            } else {
255                                // city/postcode unclear - warn if very close, otherwise only notify
256                                // TODO: get city from surrounding boundaries?
257                                if (distance < maxDistance) {
258                                    severityLevel = Severity.WARNING;
259                                } else {
260                                    severityLevel = Severity.OTHER;
261                                }
262                            }
263                        }
264                        if (severityLevel == Severity.WARNING && !Objects.equals(name1, name2)) {
265                            // since multiple objects can exist at one address, a different name tag isn't very concerning
266                            severityLevel = Severity.OTHER;
267                        }
268                        result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
269                                .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
270                                .primitives(Arrays.asList(p, p2)).build());
271                    }
272                    knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
273                }
274            }
275            errors.addAll(result);
276            return Collections.unmodifiableList(result);
277        }
278        return Collections.emptyList();
279    }
280
281    static List<String> getSimplifiedAddresses(OsmPrimitive p) {
282        String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
283        // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
284        return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of(
285                simplifiedStreetName.replaceAll("[ -]", ""),
286                addrHouseNumber,
287                p.get(ADDR_HOUSE_NAME),
288                p.get(ADDR_UNIT),
289                p.get(ADDR_FLATS))
290            .filter(Objects::nonNull)
291            .collect(Collectors.joining(" ")))
292                .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList());
293    }
294
295    /**
296     * Split addr:housenumber on , and ; (common separators)
297     *
298     * @param houseNumber The housenumber to be split
299     * @return A list of addr:housenumber equivalents
300     */
301    static List<String> expandHouseNumber(String houseNumber) {
302        return Arrays.asList(houseNumber.split("[,;]", -1));
303    }
304
305    @Override
306    public void visit(Node n) {
307        checkHouseNumbersWithoutStreet(n);
308        checkForDuplicate(n);
309    }
310
311    @Override
312    public void visit(Way w) {
313        checkHouseNumbersWithoutStreet(w);
314        checkForDuplicate(w);
315    }
316
317    @Override
318    public void visit(Relation r) {
319        checkHouseNumbersWithoutStreet(r);
320        checkForDuplicate(r);
321        if (r.hasTag("type", ASSOCIATED_STREET)) {
322            checkIfObsolete(r);
323            // Used to count occurrences of each house number in order to find duplicates
324            Map<String, List<OsmPrimitive>> map = new HashMap<>();
325            // Used to detect different street names
326            String relationName = r.get("name");
327            Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
328            // Used to check distance
329            Set<OsmPrimitive> houses = new HashSet<>();
330            Set<Way> street = new HashSet<>();
331            for (RelationMember m : r.getMembers()) {
332                String role = m.getRole();
333                OsmPrimitive p = m.getMember();
334                if (HOUSE.equals(role)) {
335                    houses.add(p);
336                    String number = p.get(ADDR_HOUSE_NUMBER);
337                    if (number != null) {
338                        number = number.trim().toUpperCase(Locale.ENGLISH);
339                        List<OsmPrimitive> list = map.computeIfAbsent(number, k -> new ArrayList<>());
340                        list.add(p);
341                    }
342                    if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
343                        if (wrongStreetNames.isEmpty()) {
344                            wrongStreetNames.add(r);
345                        }
346                        wrongStreetNames.add(p);
347                    }
348                } else if (STREET.equals(role)) {
349                    if (p instanceof Way) {
350                        street.add((Way) p);
351                    }
352                    if (relationName != null && p.hasTagDifferent("name", relationName)) {
353                        if (wrongStreetNames.isEmpty()) {
354                            wrongStreetNames.add(r);
355                        }
356                        wrongStreetNames.add(p);
357                    }
358                }
359            }
360            // Report duplicate house numbers
361            for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
362                List<OsmPrimitive> list = entry.getValue();
363                if (list.size() > 1) {
364                    errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
365                            .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
366                            .primitives(list)
367                            .build());
368                }
369            }
370            // Report wrong street names
371            if (!wrongStreetNames.isEmpty()) {
372                errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
373                        .message(tr("Multiple street names in relation"))
374                        .primitives(wrongStreetNames)
375                        .build());
376            }
377            // Report addresses too far away
378            if (!street.isEmpty()) {
379                for (OsmPrimitive house : houses) {
380                    if (house.isUsable()) {
381                        checkDistance(house, street);
382                    }
383                }
384            }
385        }
386    }
387
388    /**
389     * returns rough distance between two OsmPrimitives
390     * @param a primitive a
391     * @param b primitive b
392     * @return distance of center of bounding boxes in meters
393     */
394    static double getDistance(OsmPrimitive a, OsmPrimitive b) {
395        if (a instanceof ILatLon && b instanceof ILatLon) {
396            return ((ILatLon) a).greatCircleDistance((ILatLon) b);
397        }
398        ILatLon centerA = a.getBBox().getCenter();
399        ILatLon centerB = b.getBBox().getCenter();
400        return centerA.greatCircleDistance(centerB);
401    }
402
403    protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
404        EastNorth centroid;
405        if (house instanceof Node) {
406            centroid = ((Node) house).getEastNorth();
407        } else if (house instanceof Way) {
408            List<Node> nodes = ((Way) house).getNodes();
409            if (house.hasKey(ADDR_INTERPOLATION)) {
410                for (Node n : nodes) {
411                    if (n.hasKey(ADDR_HOUSE_NUMBER)) {
412                        checkDistance(n, street);
413                    }
414                }
415                return;
416            }
417            centroid = Geometry.getCentroid(nodes);
418        } else {
419            return; // TODO handle multipolygon houses ?
420        }
421        if (centroid == null) return; // fix #8305
422        double maxDistance = MAX_STREET_DISTANCE.get();
423        boolean hasIncompleteWays = false;
424        for (Way streetPart : street) {
425            for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
426                EastNorth p1 = chunk.a.getEastNorth();
427                EastNorth p2 = chunk.b.getEastNorth();
428                if (p1 != null && p2 != null) {
429                    EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
430                    if (closest.distance(centroid) <= maxDistance) {
431                        return;
432                    }
433                } else {
434                    Logging.warn("Addresses test skipped chunk "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
435                }
436            }
437            if (!hasIncompleteWays && streetPart.isIncomplete()) {
438                hasIncompleteWays = true;
439            }
440        }
441        // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
442        if (hasIncompleteWays) return;
443        List<OsmPrimitive> errorList = new ArrayList<>(street);
444        errorList.add(0, house);
445        errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
446                .message(tr("House number too far from street"))
447                .primitives(errorList)
448                .build());
449    }
450
451    /**
452     * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
453     * are complete and don't contain any information which isn't also tagged on the members.
454     * The strategy is to avoid any false positive.
455     * @param r the relation
456     */
457    private void checkIfObsolete(Relation r) {
458        if (r.isIncomplete())
459            return;
460        /* array of country codes for which the test should be performed. For now, only Germany */
461        String[] countryCodes = {"DE"};
462        TagMap neededtagsForHouse = new TagMap();
463        for (Entry<String, String> tag : r.getKeys().entrySet()) {
464            String key = tag.getKey();
465            if (key.startsWith("name:")) {
466                return; // maybe check if all members have corresponding tags?
467            } else if (key.startsWith("addr:")) {
468                neededtagsForHouse.put(key, tag.getValue());
469            } else {
470                switch (key) {
471                case "name":
472                case "type":
473                case "source":
474                    break;
475                default:
476                    // unexpected tag in relation
477                    return;
478                }
479            }
480        }
481
482        for (RelationMember m : r.getMembers()) {
483            if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes))
484                return;
485
486            String role = m.getRole();
487            if ("".equals(role)) {
488                if (m.isWay() && m.getMember().hasKey("highway")) {
489                    role = STREET;
490                } else if (m.getMember().hasTag("building"))
491                    role = HOUSE;
492            }
493            switch (role) {
494            case HOUSE:
495            case "addr:houselink":
496            case "address":
497                if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
498                    return;
499                for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
500                    if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
501                        return;
502                }
503                break;
504            case STREET:
505                if (!m.getMember().hasTag("name") && r.hasTag("name"))
506                    return;
507                break;
508            default:
509                // unknown role: don't create auto-fix
510                return;
511            }
512        }
513        errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
514                .message(tr("Relation is obsolete"))
515                .primitives(r)
516                .build());
517    }
518
519    private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
520        if (countryCodes.length == 0)
521            return true;
522        LatLon center = null;
523
524        if (m.isNode()) {
525            center = m.getNode().getCoor();
526        } else if (m.isWay()) {
527            center = m.getWay().getBBox().getCenter();
528        } else if (m.isRelation() && m.getRelation().isMultipolygon()) {
529            center = m.getRelation().getBBox().getCenter();
530        }
531        if (center == null)
532            return false;
533        for (String country : countryCodes) {
534            if (Territories.isIso3166Code(country, center))
535                return true;
536        }
537        return false;
538    }
539
540    /**
541     * remove obsolete relation.
542     */
543    @Override
544    public Command fixError(TestError testError) {
545        return new DeleteCommand(testError.getPrimitives());
546    }
547
548    @Override
549    public boolean isFixable(TestError testError) {
550        if (!(testError.getTester() instanceof Addresses))
551            return false;
552        return testError.getCode() == OBSOLETE_RELATION;
553    }
554
555    @Override
556    public void addGui(JPanel testPanel) {
557        super.addGui(testPanel);
558        checkboxIncludeBldgPOI.setSelected(PREF_INCLUDE_BLDG_POI.get());
559        testPanel.add(checkboxIncludeBldgPOI, GBC.eol().insets(20, 0, 0, 0));
560    }
561
562    @Override
563    public boolean ok() {
564        super.ok();
565        PREF_INCLUDE_BLDG_POI.put(checkboxIncludeBldgPOI.isSelected());
566        includeBldgAndPOI = PREF_INCLUDE_BLDG_POI.get();
567        return false;
568    }
569
570}