source: josm/trunk/test/functional/org/openstreetmap/josm/io/OsmServerBackreferenceReaderTest.java@ 4311

Last change on this file since 4311 was 4126, checked in by bastiK, 13 years ago

memory optimizations for Node & WayPoint (Patch by Gubaer, modified)

The field 'proj' in CachedLatLon is a waste of memory. For the 2 classes where this has the greatest impact, the cache for the projected coordinates is replaced by 2 simple double fields (east & north). On projection change, they have to be invalidated explicitly. This is handled by the DataSet & the GpxLayer.
File size: 19.7 KB
Line 
1// License: GPL. For details, see LICENSE file.
2package org.openstreetmap.josm.io;
3
4import static org.junit.Assert.assertEquals;
5import static org.junit.Assert.assertNotNull;
6import static org.junit.Assert.fail;
7
8import java.io.File;
9import java.io.FileInputStream;
10import java.io.FileWriter;
11import java.io.IOException;
12import java.io.PrintWriter;
13import java.text.MessageFormat;
14import java.util.ArrayList;
15import java.util.HashSet;
16import java.util.Properties;
17import java.util.Set;
18import java.util.logging.Level;
19import java.util.logging.Logger;
20
21import org.junit.Before;
22import org.junit.BeforeClass;
23import org.junit.Test;
24import org.openstreetmap.josm.Main;
25import org.openstreetmap.josm.data.coor.LatLon;
26import org.openstreetmap.josm.data.osm.Changeset;
27import org.openstreetmap.josm.data.osm.DataSet;
28import org.openstreetmap.josm.data.osm.Node;
29import org.openstreetmap.josm.data.osm.OsmPrimitive;
30import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
31import org.openstreetmap.josm.data.osm.Relation;
32import org.openstreetmap.josm.data.osm.RelationMember;
33import org.openstreetmap.josm.data.osm.Way;
34import org.openstreetmap.josm.data.projection.Mercator;
35import org.openstreetmap.josm.gui.io.UploadStrategy;
36import org.openstreetmap.josm.gui.io.UploadStrategySpecification;
37import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
38
39public class OsmServerBackreferenceReaderTest {
40 static private final Logger logger = Logger.getLogger(OsmServerBackreferenceReader.class.getName());
41
42 protected static Node lookupNode(DataSet ds, int i) {
43 for (Node n : ds.getNodes()) {
44 if (("node-" + i).equals(n.get("name"))) return n;
45 }
46 return null;
47 }
48
49
50 protected static Way lookupWay(DataSet ds, int i) {
51 for (Way w : ds.getWays()) {
52 if (("way-" + i).equals(w.get("name"))) return w;
53 }
54 return null;
55 }
56
57 protected static Relation lookupRelation(DataSet ds, int i) {
58 for (Relation r : ds.getRelations()) {
59 if (("relation-" + i).equals(r.get("name"))) return r;
60 }
61 return null;
62 }
63
64 protected static void populateTestDataSetWithNodes(DataSet ds) {
65 for (int i=0;i<100;i++) {
66 Node n = new Node();
67 n.setCoor(new LatLon(-36.6,47.6));
68 n.put("name", "node-"+i);
69 ds.addPrimitive(n);
70 }
71 }
72
73 protected static void populateTestDataSetWithWays(DataSet ds) {
74 for (int i=0;i<20;i++) {
75 Way w = new Way();
76 for (int j = 0; j < 10;j++) {
77 w.addNode(lookupNode(ds, i+j));
78 }
79 w.put("name", "way-"+i);
80 ds.addPrimitive(w);
81 }
82 }
83
84 protected static void populateTestDataSetWithRelations(DataSet ds) {
85 for (int i=0;i<10;i++) {
86 Relation r = new Relation();
87 r.put("name", "relation-" +i);
88 for (int j =0; j < 10; j++) {
89 RelationMember member = new RelationMember("node-" + j, lookupNode(ds, i + j));
90 r.addMember(member);
91 }
92 for (int j =0; j < 5; j++) {
93 RelationMember member = new RelationMember("way-" + j, lookupWay(ds, i + j));
94 r.addMember(member);
95 }
96 if (i > 5) {
97 for (int j =0; j < 3; j++) {
98 RelationMember member = new RelationMember("relation-" + j, lookupRelation(ds, j));
99 logger.info(MessageFormat.format("adding relation {0} to relation {1}", j, i));
100 r.addMember(member);
101 }
102 }
103 ds.addPrimitive(r);
104 }
105 }
106
107
108 protected static DataSet buildTestDataSet() {
109 DataSet ds = new DataSet();
110 ds.setVersion("0.6");
111
112 populateTestDataSetWithNodes(ds);
113 populateTestDataSetWithWays(ds);
114 populateTestDataSetWithRelations(ds);
115 return ds;
116 }
117
118 /**
119 * creates the dataset on the server.
120 *
121 * @param ds the data set
122 * @throws OsmTransferException
123 */
124 static public void createDataSetOnServer(DataSet ds) throws OsmTransferException {
125 logger.info("creating data set on the server ...");
126 ArrayList<OsmPrimitive> primitives = new ArrayList<OsmPrimitive>();
127 primitives.addAll(ds.getNodes());
128 primitives.addAll(ds.getWays());
129 primitives.addAll(ds.getRelations());
130 OsmServerWriter writer = new OsmServerWriter();
131 Changeset cs = new Changeset();
132 writer.uploadOsm(new UploadStrategySpecification().setStrategy(UploadStrategy.SINGLE_REQUEST_STRATEGY), primitives, cs, NullProgressMonitor.INSTANCE);
133 OsmApi.getOsmApi().closeChangeset(cs, NullProgressMonitor.INSTANCE);
134 }
135
136 static Properties testProperties;
137 static DataSet testDataSet;
138
139 @BeforeClass
140 public static void init() throws OsmTransferException {
141 logger.info("initializing ...");
142 testProperties = new Properties();
143
144 // load properties
145 //
146 try {
147 testProperties.load(MultiFetchServerObjectReaderTest.class.getResourceAsStream("/test-functional-env.properties"));
148 } catch(Exception e){
149 logger.log(Level.SEVERE, MessageFormat.format("failed to load property file ''{0}''", "test-functional-env.properties"));
150 fail(MessageFormat.format("failed to load property file ''{0}''", "test-functional-env.properties"));
151 }
152
153 // check josm.home
154 //
155 String josmHome = testProperties.getProperty("josm.home");
156 if (josmHome == null) {
157 fail(MessageFormat.format("property ''{0}'' not set in test environment", "josm.home"));
158 } else {
159 File f = new File(josmHome);
160 if (! f.exists() || ! f.canRead()) {
161 fail(MessageFormat.format("property ''{0}'' points to ''{1}'' which is either not existing or not readable", "josm.home", josmHome));
162 }
163 }
164
165 // check temp output dir
166 //
167 String tempOutputDir = testProperties.getProperty("test.functional.tempdir");
168 if (tempOutputDir == null) {
169 fail(MessageFormat.format("property ''{0}'' not set in test environment", "test.functional.tempdir"));
170 } else {
171 File f = new File(tempOutputDir);
172 if (! f.exists() || ! f.isDirectory() || ! f.canWrite()) {
173 fail(MessageFormat.format("property ''{0}'' points to ''{1}'' which is either not existing, not a directory, or not writeable", "test.functional.tempdir", tempOutputDir));
174 }
175 }
176
177
178 // init preferences
179 //
180 System.setProperty("josm.home", josmHome);
181 Main.pref.init(false);
182 // don't use atomic upload, the test API server can't cope with large diff uploads
183 //
184 Main.pref.put("osm-server.atomic-upload", false);
185 Main.setProjection(new Mercator());
186
187 File dataSetCacheOutputFile = new File(tempOutputDir, MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
188
189 // make sure we don't upload to production
190 //
191 String url = OsmApi.getOsmApi().getBaseUrl().toLowerCase().trim();
192 if (url.startsWith("http://www.openstreetmap.org")
193 || url.startsWith("http://api.openstreetmap.org")) {
194 fail(MessageFormat.format("configured url ''{0}'' seems to be a productive url, aborting.", url));
195 }
196
197
198 String p = System.getProperties().getProperty("useCachedDataset");
199 if (p != null && Boolean.parseBoolean(p.trim().toLowerCase())) {
200 logger.info(MessageFormat.format("property ''{0}'' set, using cached dataset", "useCachedDataset"));
201 return;
202 }
203
204 logger.info(MessageFormat.format("property ''{0}'' not set to true, creating test dataset on the server. property is ''{1}''", "useCachedDataset", p));
205
206 // build and upload the test data set
207 //
208 logger.info("creating test data set ....");
209 testDataSet = buildTestDataSet();
210 logger.info("uploading test data set ...");
211 createDataSetOnServer(testDataSet);
212
213 try {
214 PrintWriter pw = new PrintWriter(
215 new FileWriter(dataSetCacheOutputFile)
216 );
217 logger.info(MessageFormat.format("caching test data set in ''{0}'' ...", dataSetCacheOutputFile.toString()));
218 OsmWriter w = new OsmWriter(pw, false, testDataSet.getVersion());
219 w.header();
220 w.writeDataSources(testDataSet);
221 w.writeContent(testDataSet);
222 w.footer();
223 w.close();
224 pw.close();
225 } catch(IOException e) {
226 fail(MessageFormat.format("failed to open file ''{0}'' for writing", dataSetCacheOutputFile.toString()));
227 }
228 }
229
230 private DataSet ds;
231
232 @Before
233 public void setUp() throws IOException, IllegalDataException {
234 File f = new File(testProperties.getProperty("test.functional.tempdir"), MultiFetchServerObjectReaderTest.class.getName() + ".dataset");
235 logger.info(MessageFormat.format("reading cached dataset ''{0}''", f.toString()));
236 ds = new DataSet();
237 FileInputStream fis = new FileInputStream(f);
238 ds = OsmReader.parseDataSet(fis, NullProgressMonitor.INSTANCE);
239 fis.close();
240 }
241
242 @Test
243 public void testBackrefrenceForNode() throws OsmTransferException {
244 Node n = lookupNode(ds, 0);
245 assertNotNull(n);
246 Way w = lookupWay(ds, 0);
247 assertNotNull(w);
248
249 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
250 reader.setReadFull(false);
251 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
252 assertEquals(10, referers.getNodes().size());
253 assertEquals(1, referers.getWays().size());
254 assertEquals(0, referers.getRelations().size());
255 for (Way way : referers.getWays()) {
256 assertEquals(w.getId(), way.getId());
257 assertEquals(false, way.isIncomplete());
258 }
259 }
260
261 @Test
262 public void testBackrefrenceForNode_Full() throws OsmTransferException {
263 Node n = lookupNode(ds, 0);
264 assertNotNull(n);
265 Way w = lookupWay(ds, 0);
266 assertNotNull(w);
267
268 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(n);
269 reader.setReadFull(true);
270 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
271 assertEquals(10, referers.getNodes().size());
272 assertEquals(1, referers.getWays().size());
273 assertEquals(0, referers.getRelations().size());
274 for (Way way : referers.getWays()) {
275 assertEquals(w.getId(), way.getId());
276 assertEquals(false, way.isIncomplete());
277 assertEquals(10, w.getNodesCount());
278 }
279 }
280
281 @Test
282 public void testBackrefrenceForWay() throws OsmTransferException {
283 Way w = lookupWay(ds, 1);
284 assertNotNull(w);
285 // way with name "way-1" is referred to by two relations
286 //
287
288 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
289 reader.setReadFull(false);
290 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
291 assertEquals(0, referers.getNodes().size()); // no nodes loaded
292 assertEquals(6, referers.getWays().size()); // 6 ways referred by two relations
293 for (Way w1 : referers.getWays()) {
294 assertEquals(true, w1.isIncomplete());
295 }
296 assertEquals(2, referers.getRelations().size()); // two relations referring to w
297
298 Relation r = lookupRelation(referers, 0);
299 assertNotNull(r);
300 assertEquals(false, r.isIncomplete());
301 r = lookupRelation(referers, 1);
302 assertEquals(false, r.isIncomplete());
303 }
304
305 @Test
306 public void testBackrefrenceForWay_Full() throws OsmTransferException {
307 Way w = lookupWay(ds, 1);
308 assertNotNull(w);
309 // way with name "way-1" is referred to by two relations
310 //
311
312 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(w);
313 reader.setReadFull(true);
314 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
315 assertEquals(6, referers.getWays().size()); // 6 ways referred by two relations
316 for (Way w1 : referers.getWays()) {
317 assertEquals(false, w1.isIncomplete());
318 }
319 assertEquals(2, referers.getRelations().size()); // two relations referring to
320 Set<Long> expectedNodeIds = new HashSet<Long>();
321 for (Way way : referers.getWays()) {
322 Way orig = (Way) ds.getPrimitiveById(way);
323 for (Node n : orig.getNodes()) {
324 expectedNodeIds.add(n.getId());
325 }
326 }
327 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
328 for (Node n : referers.getNodes()) {
329 assertEquals(true, expectedNodeIds.contains(n.getId()));
330 }
331
332 Relation r = lookupRelation(referers, 0);
333 assertNotNull(r);
334 assertEquals(false, r.isIncomplete());
335 r = lookupRelation(referers, 1);
336 assertEquals(false, r.isIncomplete());
337 }
338
339 @Test
340 public void testBackrefrenceForRelation() throws OsmTransferException {
341 Relation r = lookupRelation(ds, 1);
342 assertNotNull(r);
343 // way with name "relation-1" is referred to by four relations:
344 // relation-6, relation-7, relation-8, relation-9
345 //
346
347 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
348 reader.setReadFull(false);
349 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
350
351 Set<Long> referringRelationsIds = new HashSet<Long>();
352 r = lookupRelation(referers, 6);
353 assertNotNull(r);
354 assertEquals(false, r.isIncomplete());
355 referringRelationsIds.add(r.getId());
356 r = lookupRelation(referers, 7);
357 assertNotNull(r);
358 assertEquals(false, r.isIncomplete());
359 referringRelationsIds.add(r.getId());
360 r = lookupRelation(referers, 8);
361 assertNotNull(r);
362 assertEquals(false, r.isIncomplete());
363 referringRelationsIds.add(r.getId());
364 r = lookupRelation(referers, 9);
365 assertNotNull(r);
366 assertEquals(false, r.isIncomplete());
367 referringRelationsIds.add(r.getId());
368
369 for (Relation r1 : referers.getRelations()) {
370 if (!referringRelationsIds.contains(r1.getId())) {
371 assertEquals(true, r1.isIncomplete());
372 }
373 }
374
375 // make sure we read all ways referred to by parent relations. These
376 // ways are incomplete after reading.
377 //
378 Set<Long> expectedWayIds = new HashSet<Long>();
379 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
380 if (m.isWay()) {
381 expectedWayIds.add(m.getMember().getId());
382 }
383 }
384 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
385 if (m.isWay()) {
386 expectedWayIds.add(m.getMember().getId());
387 }
388 }
389 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
390 if (m.isWay()) {
391 expectedWayIds.add(m.getMember().getId());
392 }
393 }
394 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
395 if (m.isWay()) {
396 expectedWayIds.add(m.getMember().getId());
397 }
398 }
399
400 assertEquals(expectedWayIds.size(), referers.getWays().size());
401 for (Way w1 : referers.getWays()) {
402 assertEquals(true, expectedWayIds.contains(w1.getId()));
403 assertEquals(true, w1.isIncomplete());
404 }
405
406 // make sure we didn't read any nodes
407 //
408 assertEquals(0, referers.getNodes().size());
409 }
410
411 protected Set<Long> getNodeIdsInWay(Way way) {
412 HashSet<Long> ret = new HashSet<Long>();
413 if (way == null)return ret;
414 for (Node n: way.getNodes()) {
415 ret.add(n.getId());
416 }
417 return ret;
418 }
419
420 protected Set<Long> getNodeIdsInRelation(Relation r) {
421 HashSet<Long> ret = new HashSet<Long>();
422 if (r == null) return ret;
423 for (RelationMember m: r.getMembers()) {
424 if (m.isNode()) {
425 ret.add(m.getMember().getId());
426 } else if (m.isWay()) {
427 ret.addAll(getNodeIdsInWay(m.getWay()));
428 } else if (m.isRelation()) {
429 ret.addAll(getNodeIdsInRelation(m.getRelation()));
430 }
431 }
432 return ret;
433 }
434
435 @Test
436 public void testBackrefrenceForRelation_Full() throws OsmTransferException {
437 Relation r = lookupRelation(ds, 1);
438 assertNotNull(r);
439 // way with name "relation-1" is referred to by four relations:
440 // relation-6, relation-7, relation-8, relation-9
441 //
442
443 OsmServerBackreferenceReader reader = new OsmServerBackreferenceReader(r);
444 reader.setReadFull(true);
445 DataSet referers = reader.parseOsm(NullProgressMonitor.INSTANCE);
446
447 Set<Long> referringRelationsIds = new HashSet<Long>();
448 r = lookupRelation(referers, 6);
449 assertNotNull(r);
450 assertEquals(false, r.isIncomplete());
451 referringRelationsIds.add(r.getId());
452 r = lookupRelation(referers, 7);
453 assertNotNull(r);
454 assertEquals(false, r.isIncomplete());
455 referringRelationsIds.add(r.getId());
456 r = lookupRelation(referers, 8);
457 assertNotNull(r);
458 assertEquals(false, r.isIncomplete());
459 referringRelationsIds.add(r.getId());
460 r = lookupRelation(referers, 9);
461 assertNotNull(r);
462 assertEquals(false, r.isIncomplete());
463 referringRelationsIds.add(r.getId());
464
465 // all relations are fully loaded
466 //
467 for (Relation r1 : referers.getRelations()) {
468 assertEquals(false, r1.isIncomplete());
469 }
470
471 // make sure we read all ways referred to by parent relations. These
472 // ways are completely read after reading the relations
473 //
474 Set<Long> expectedWayIds = new HashSet<Long>();
475 for (RelationMember m : lookupRelation(ds, 6).getMembers()) {
476 if (m.isWay()) {
477 expectedWayIds.add(m.getMember().getId());
478 }
479 }
480 for (RelationMember m : lookupRelation(ds, 7).getMembers()) {
481 if (m.isWay()) {
482 expectedWayIds.add(m.getMember().getId());
483 }
484 }
485 for (RelationMember m : lookupRelation(ds, 8).getMembers()) {
486 if (m.isWay()) {
487 expectedWayIds.add(m.getMember().getId());
488 }
489 }
490 for (RelationMember m : lookupRelation(ds, 9).getMembers()) {
491 if (m.isWay()) {
492 expectedWayIds.add(m.getMember().getId());
493 }
494 }
495 for (long id : expectedWayIds) {
496 Way w = (Way) referers.getPrimitiveById(id, OsmPrimitiveType.WAY);
497 assertNotNull(w);
498 assertEquals(false, w.isIncomplete());
499 }
500
501 Set<Long> expectedNodeIds = new HashSet<Long>();
502 for (int i = 6; i < 10; i++) {
503 Relation r1 = lookupRelation(ds, i);
504 expectedNodeIds.addAll(getNodeIdsInRelation(r1));
505 }
506
507 assertEquals(expectedNodeIds.size(), referers.getNodes().size());
508 for (Node n : referers.getNodes()) {
509 assertEquals(true, expectedNodeIds.contains(n.getId()));
510 }
511 }
512}
Note: See TracBrowser for help on using the repository browser.