Index: trunk/src/org/openstreetmap/josm/gui/layer/AutosaveTask.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/layer/AutosaveTask.java	(revision 17046)
+++ trunk/src/org/openstreetmap/josm/gui/layer/AutosaveTask.java	(revision 17047)
@@ -316,4 +316,8 @@
                 noteLayer.getNoteData().removeNoteDataUpdateListener(this);
                 cleanupLayer(noteLayer);
+            }
+        } else if (e.getRemovedLayer() instanceof AbstractModifiableLayer) {
+            synchronized (layersLock) {
+                cleanupLayer((AbstractModifiableLayer) e.getRemovedLayer());
             }
         }
Index: trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java	(revision 17046)
+++ trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java	(revision 17047)
@@ -32,4 +32,6 @@
 import org.openstreetmap.josm.gui.layer.GpxLayer;
 import org.openstreetmap.josm.gui.preferences.display.GPXSettingsPanel;
+import org.openstreetmap.josm.spi.preferences.PreferenceChangedListener;
+import org.openstreetmap.josm.tools.Destroyable;
 import org.openstreetmap.josm.tools.ImageProvider;
 import org.openstreetmap.josm.tools.Logging;
@@ -74,5 +76,5 @@
  * @author Frederik Ramm
  */
-public class Marker implements TemplateEngineDataProvider, ILatLon {
+public class Marker implements TemplateEngineDataProvider, ILatLon, Destroyable {
 
     /**
@@ -148,4 +150,5 @@
 
     private CachedLatLon coor;
+    private PreferenceChangedListener listener = l -> updateText();
 
     private boolean erroneous;
@@ -175,5 +178,5 @@
         this.text = text;
 
-        Preferences.main().addKeyPreferenceChangeListener("draw.rawgps." + getTextTemplateKey(), l -> updateText());
+        Preferences.main().addKeyPreferenceChangeListener(getPreferenceKey(), listener);
     }
 
@@ -370,5 +373,5 @@
         cachedText = null;
         cachedDefaultTemplate = null;
-        cachedTemplates = new HashMap<>();
+        cachedTemplates.clear();
     }
 
@@ -433,3 +436,13 @@
         }
     }
+
+    @Override
+    public void destroy() {
+        cachedTemplates.clear();
+        Preferences.main().removeKeyPreferenceChangeListener(getPreferenceKey(), listener);
+    }
+
+    private String getPreferenceKey() {
+        return "draw.rawgps." + getTextTemplateKey();
+    }
 }
Index: trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(revision 17046)
+++ trunk/src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(revision 17047)
@@ -172,4 +172,5 @@
         currentMarker = null;
         fromLayer = null;
+        data.forEach(Marker::destroy);
         data.clear();
         super.destroy();