Index: trunk/src/org/openstreetmap/josm/gui/dialogs/NotesDialog.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/dialogs/NotesDialog.java	(revision 15382)
+++ trunk/src/org/openstreetmap/josm/gui/dialogs/NotesDialog.java	(revision 15385)
@@ -14,4 +14,5 @@
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 
 import javax.swing.AbstractAction;
@@ -155,10 +156,13 @@
     public void layerRemoving(LayerRemoveEvent e) {
         if (e.getRemovedLayer() instanceof NoteLayer) {
-            noteData.removeNoteDataUpdateListener(this);
-            noteData = null;
-            model.clearData();
-            MapFrame map = MainApplication.getMap();
-            if (map.mapMode instanceof AddNoteAction) {
-                map.selectMapMode(map.mapModeSelect);
+            NoteData removedNoteData = ((NoteLayer) e.getRemovedLayer()).getNoteData();
+            removedNoteData.removeNoteDataUpdateListener(this);
+            if (Objects.equals(noteData, removedNoteData)) {
+                noteData = null;
+                model.clearData();
+                MapFrame map = MainApplication.getMap();
+                if (map.mapMode instanceof AddNoteAction) {
+                    map.selectMapMode(map.mapModeSelect);
+                }
             }
         }