source: josm/trunk/scripts/TagInfoExtract.java@ 15043

Last change on this file since 15043 was 14746, checked in by simon04, 7 years ago

Refactoring: use StandardCharsets
File size: 23.8 KB
Line 
1// License: GPL. For details, see LICENSE file.
2
3import java.awt.Graphics2D;
4import java.awt.image.BufferedImage;
5import java.io.BufferedReader;
6import java.io.IOException;
7import java.io.OutputStream;
8import java.io.StringWriter;
9import java.io.UncheckedIOException;
10import java.io.Writer;
11import java.nio.file.Files;
12import java.nio.file.Path;
13import java.nio.file.Paths;
14import java.time.Instant;
15import java.time.ZoneId;
16import java.time.format.DateTimeFormatter;
17import java.util.ArrayList;
18import java.util.Arrays;
19import java.util.Collection;
20import java.util.Collections;
21import java.util.EnumSet;
22import java.util.List;
23import java.util.Locale;
24import java.util.Optional;
25import java.util.Set;
26import java.util.stream.Collectors;
27
28import javax.imageio.ImageIO;
29import javax.json.Json;
30import javax.json.JsonArrayBuilder;
31import javax.json.JsonObjectBuilder;
32import javax.json.JsonWriter;
33import javax.json.stream.JsonGenerator;
34
35import org.openstreetmap.josm.actions.DeleteAction;
36import org.openstreetmap.josm.command.DeleteCommand;
37import org.openstreetmap.josm.data.Preferences;
38import org.openstreetmap.josm.data.Version;
39import org.openstreetmap.josm.data.coor.LatLon;
40import org.openstreetmap.josm.data.osm.Node;
41import org.openstreetmap.josm.data.osm.OsmPrimitive;
42import org.openstreetmap.josm.data.osm.Tag;
43import org.openstreetmap.josm.data.osm.Way;
44import org.openstreetmap.josm.data.osm.visitor.paint.MapPaintSettings;
45import org.openstreetmap.josm.data.osm.visitor.paint.StyledMapRenderer;
46import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
47import org.openstreetmap.josm.data.preferences.JosmUrls;
48import org.openstreetmap.josm.data.preferences.sources.ExtendedSourceEntry;
49import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
50import org.openstreetmap.josm.data.projection.ProjectionRegistry;
51import org.openstreetmap.josm.data.projection.Projections;
52import org.openstreetmap.josm.gui.NavigatableComponent;
53import org.openstreetmap.josm.gui.mappaint.Cascade;
54import org.openstreetmap.josm.gui.mappaint.Environment;
55import org.openstreetmap.josm.gui.mappaint.MapPaintStyles;
56import org.openstreetmap.josm.gui.mappaint.MultiCascade;
57import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory;
58import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
59import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
60import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
61import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
62import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
63import org.openstreetmap.josm.gui.mappaint.styleelement.AreaElement;
64import org.openstreetmap.josm.gui.mappaint.styleelement.LineElement;
65import org.openstreetmap.josm.gui.mappaint.styleelement.StyleElement;
66import org.openstreetmap.josm.gui.preferences.map.TaggingPresetPreference;
67import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
68import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetReader;
69import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
70import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
71import org.openstreetmap.josm.io.CachedFile;
72import org.openstreetmap.josm.io.OsmTransferException;
73import org.openstreetmap.josm.spi.preferences.Config;
74import org.openstreetmap.josm.tools.Logging;
75import org.openstreetmap.josm.tools.OptionParser;
76import org.openstreetmap.josm.tools.RightAndLefthandTraffic;
77import org.openstreetmap.josm.tools.Territories;
78import org.openstreetmap.josm.tools.Utils;
79import org.xml.sax.SAXException;
80
81/**
82 * Extracts tag information for the taginfo project.
83 * <p>
84 * Run from the base directory of a JOSM checkout:
85 * <p>
86 * java -cp dist/josm-custom.jar TagInfoExtract --type mappaint
87 * java -cp dist/josm-custom.jar TagInfoExtract --type presets
88 * java -cp dist/josm-custom.jar TagInfoExtract --type external_presets
89 */
90public class TagInfoExtract {
91
92 /**
93 * Main method.
94 */
95 public static void main(String[] args) throws Exception {
96 TagInfoExtract script = new TagInfoExtract();
97 script.parseCommandLineArguments(args);
98 script.init();
99 switch (script.options.mode) {
100 case MAPPAINT:
101 script.new StyleSheet().run();
102 break;
103 case PRESETS:
104 script.new Presets().run();
105 break;
106 case EXTERNAL_PRESETS:
107 script.new ExternalPresets().run();
108 break;
109 default:
110 throw new IllegalStateException("Invalid type " + script.options.mode);
111 }
112 if (!script.options.noexit) {
113 System.exit(0);
114 }
115 }
116
117 enum Mode {
118 MAPPAINT, PRESETS, EXTERNAL_PRESETS
119 }
120
121 private final Options options = new Options();
122
123 /**
124 * Parse command line arguments.
125 */
126 private void parseCommandLineArguments(String[] args) {
127 if (args.length == 1 && "--help".equals(args[0])) {
128 this.usage();
129 }
130 final OptionParser parser = new OptionParser(getClass().getName());
131 parser.addArgumentParameter("type", OptionParser.OptionCount.REQUIRED, options::setMode);
132 parser.addArgumentParameter("input", OptionParser.OptionCount.OPTIONAL, options::setInputFile);
133 parser.addArgumentParameter("output", OptionParser.OptionCount.OPTIONAL, options::setOutputFile);
134 parser.addArgumentParameter("imgdir", OptionParser.OptionCount.OPTIONAL, options::setImageDir);
135 parser.addArgumentParameter("imgurlprefix", OptionParser.OptionCount.OPTIONAL, options::setImageUrlPrefix);
136 parser.addFlagParameter("noexit", options::setNoExit);
137 parser.addFlagParameter("help", this::usage);
138 parser.parseOptionsOrExit(Arrays.asList(args));
139 }
140
141 private void usage() {
142 System.out.println("java " + getClass().getName());
143 System.out.println(" --type TYPE\tthe project type to be generated: " + Arrays.toString(Mode.values()));
144 System.out.println(" --input FILE\tthe input file to use (overrides defaults for types mappaint, presets)");
145 System.out.println(" --output FILE\tthe output file to use (defaults to STDOUT)");
146 System.out.println(" --imgdir DIRECTORY\tthe directory to put the generated images in (default: " + options.imageDir + ")");
147 System.out.println(" --imgurlprefix STRING\timage URLs prefix for generated image files (public path on webserver)");
148 System.out.println(" --noexit\tdo not call System.exit(), for use from Ant script");
149 System.out.println(" --help\tshow this help");
150 System.exit(0);
151 }
152
153 private static class Options {
154 Mode mode;
155 int josmSvnRevision = Version.getInstance().getVersion();
156 Path baseDir = Paths.get("");
157 Path imageDir = Paths.get("taginfo-img");
158 String imageUrlPrefix;
159 CachedFile inputFile;
160 Path outputFile;
161 boolean noexit;
162
163 void setMode(String value) {
164 mode = Mode.valueOf(value.toUpperCase(Locale.ENGLISH));
165 switch (mode) {
166 case MAPPAINT:
167 inputFile = new CachedFile("resource://styles/standard/elemstyles.mapcss");
168 break;
169 case PRESETS:
170 inputFile = new CachedFile("resource://data/defaultpresets.xml");
171 break;
172 default:
173 inputFile = null;
174 }
175 }
176
177 void setInputFile(String value) {
178 inputFile = new CachedFile(value);
179 }
180
181 void setOutputFile(String value) {
182 outputFile = Paths.get(value);
183 }
184
185 void setImageDir(String value) {
186 imageDir = Paths.get(value);
187 }
188
189 void setImageUrlPrefix(String value) {
190 imageUrlPrefix = value;
191 }
192
193 void setNoExit() {
194 noexit = true;
195 }
196
197 /**
198 * Determine full image url (can refer to JOSM or OSM repository).
199 * @param path the image path
200 */
201 private String findImageUrl(String path) {
202 final Path f = baseDir.resolve("images").resolve(path);
203 if (Files.exists(f)) {
204 return "https://josm.openstreetmap.de/export/" + josmSvnRevision + "/josm/trunk/images/" + path;
205 }
206 throw new IllegalStateException("Cannot find image url for " + path);
207 }
208 }
209
210 private abstract class Extractor {
211 abstract void run() throws Exception;
212
213 void writeJson(String name, String description, Iterable<TagInfoTag> tags) throws IOException {
214 try (Writer writer = options.outputFile != null ? Files.newBufferedWriter(options.outputFile) : new StringWriter();
215 JsonWriter json = Json
216 .createWriterFactory(Collections.singletonMap(JsonGenerator.PRETTY_PRINTING, true))
217 .createWriter(writer)) {
218 JsonObjectBuilder project = Json.createObjectBuilder()
219 .add("name", name)
220 .add("description", description)
221 .add("project_url", "https://josm.openstreetmap.de/")
222 .add("icon_url", "https://josm.openstreetmap.de/export/7770/josm/trunk/images/logo_16x16x8.png")
223 .add("contact_name", "JOSM developer team")
224 .add("contact_email", "josm-dev@openstreetmap.org");
225 final JsonArrayBuilder jsonTags = Json.createArrayBuilder();
226 for (TagInfoTag t : tags) {
227 jsonTags.add(t.toJson());
228 }
229 json.writeObject(Json.createObjectBuilder()
230 .add("data_format", 1)
231 .add("data_updated", DateTimeFormatter.ofPattern("yyyyMMdd'T'hhmmss'Z'").withZone(ZoneId.of("Z")).format(Instant.now()))
232 .add("project", project)
233 .add("tags", jsonTags)
234 .build());
235 if (options.outputFile == null) {
236 System.out.println(writer.toString());
237 }
238 }
239 }
240
241 }
242
243 private class Presets extends Extractor {
244
245 @Override
246 void run() throws IOException, OsmTransferException, SAXException {
247 try (BufferedReader reader = options.inputFile.getContentReader()) {
248 Collection<TaggingPreset> presets = TaggingPresetReader.readAll(reader, true);
249 List<TagInfoTag> tags = convertPresets(presets, "", true);
250 writeJson("JOSM main presets", "Tags supported by the default presets in the OSM editor JOSM", tags);
251 }
252 }
253
254 List<TagInfoTag> convertPresets(Iterable<TaggingPreset> presets, String descriptionPrefix, boolean addImages) {
255 final List<TagInfoTag> tags = new ArrayList<>();
256 for (TaggingPreset preset : presets) {
257 for (KeyedItem item : Utils.filteredCollection(preset.data, KeyedItem.class)) {
258 final Iterable<String> values = item.isKeyRequired()
259 ? item.getValues()
260 : Collections.emptyList();
261 for (String value : values) {
262 final Set<TagInfoTag.Type> types = preset.types == null ? Collections.emptySet() : preset.types.stream()
263 .map(it -> TaggingPresetType.CLOSEDWAY.equals(it)
264 ? TagInfoTag.Type.AREA
265 : TaggingPresetType.MULTIPOLYGON.equals(it)
266 ? TagInfoTag.Type.RELATION
267 : TagInfoTag.Type.valueOf(it.toString()))
268 .collect(Collectors.toCollection(() -> EnumSet.noneOf(TagInfoTag.Type.class)));
269 tags.add(new TagInfoTag(descriptionPrefix + preset.getName(), item.key, value, types,
270 addImages && preset.iconName != null ? options.findImageUrl(preset.iconName) : null));
271 }
272 }
273 }
274 return tags;
275 }
276
277 }
278
279 private class ExternalPresets extends Presets {
280
281 @Override
282 void run() throws IOException, OsmTransferException, SAXException {
283 TaggingPresetReader.setLoadIcons(false);
284 final Collection<ExtendedSourceEntry> sources = new TaggingPresetPreference.TaggingPresetSourceEditor().loadAndGetAvailableSources();
285 final List<TagInfoTag> tags = new ArrayList<>();
286 for (SourceEntry source : sources) {
287 if (source.url.startsWith("resource")) {
288 // default presets
289 continue;
290 }
291 try {
292 System.out.println("Loading " + source.url);
293 Collection<TaggingPreset> presets = TaggingPresetReader.readAll(source.url, false);
294 final List<TagInfoTag> t = convertPresets(presets, source.title + " ", false);
295 System.out.println("Converting " + t.size() + " presets of " + source.title);
296 tags.addAll(t);
297 } catch (Exception ex) {
298 System.err.println("Skipping " + source.url + " due to error");
299 ex.printStackTrace();
300 }
301
302 }
303 writeJson("JOSM user presets", "Tags supported by the user contributed presets in the OSM editor JOSM", tags);
304 }
305 }
306
307 private class StyleSheet extends Extractor {
308 private MapCSSStyleSource styleSource;
309
310 @Override
311 void run() throws IOException, ParseException {
312 init();
313 parseStyleSheet();
314 final List<TagInfoTag> tags = convertStyleSheet();
315 writeJson("JOSM main mappaint style", "Tags supported by the main mappaint style in the OSM editor JOSM", tags);
316 }
317
318 /**
319 * Read the style sheet file and parse the MapCSS code.
320 */
321 private void parseStyleSheet() throws IOException, ParseException {
322 try (BufferedReader reader = options.inputFile.getContentReader()) {
323 MapCSSParser parser = new MapCSSParser(reader, MapCSSParser.LexicalState.DEFAULT);
324 styleSource = new MapCSSStyleSource("");
325 styleSource.url = "";
326 parser.sheet(styleSource);
327 }
328 }
329
330 /**
331 * Collect all the tag from the style sheet.
332 */
333 private List<TagInfoTag> convertStyleSheet() {
334 return styleSource.rules.stream()
335 .map(rule -> rule.selector)
336 .filter(Selector.GeneralSelector.class::isInstance)
337 .map(Selector.GeneralSelector.class::cast)
338 .map(Selector.AbstractSelector::getConditions)
339 .flatMap(Collection::stream)
340 .filter(ConditionFactory.SimpleKeyValueCondition.class::isInstance)
341 .map(ConditionFactory.SimpleKeyValueCondition.class::cast)
342 .map(condition -> condition.asTag(null))
343 .distinct()
344 .map(tag -> {
345 String iconUrl = null;
346 final EnumSet<TagInfoTag.Type> types = EnumSet.noneOf(TagInfoTag.Type.class);
347 Optional<String> nodeUrl = new NodeChecker(tag).findUrl(true);
348 if (nodeUrl.isPresent()) {
349 iconUrl = nodeUrl.get();
350 types.add(TagInfoTag.Type.NODE);
351 }
352 Optional<String> wayUrl = new WayChecker(tag).findUrl(iconUrl == null);
353 if (wayUrl.isPresent()) {
354 if (iconUrl == null) {
355 iconUrl = wayUrl.get();
356 }
357 types.add(TagInfoTag.Type.WAY);
358 }
359 Optional<String> areaUrl = new AreaChecker(tag).findUrl(iconUrl == null);
360 if (areaUrl.isPresent()) {
361 if (iconUrl == null) {
362 iconUrl = areaUrl.get();
363 }
364 types.add(TagInfoTag.Type.AREA);
365 }
366 return new TagInfoTag(null, tag.getKey(), tag.getValue(), types, iconUrl);
367 })
368 .collect(Collectors.toList());
369 }
370
371 /**
372 * Check if a certain tag is supported by the style as node / way / area.
373 */
374 private abstract class Checker {
375 Checker(Tag tag) {
376 this.tag = tag;
377 }
378
379 Environment applyStylesheet(OsmPrimitive osm) {
380 osm.put(tag);
381 MultiCascade mc = new MultiCascade();
382
383 Environment env = new Environment(osm, mc, null, styleSource);
384 for (MapCSSRule r : styleSource.rules) {
385 env.clearSelectorMatchingInformation();
386 if (r.selector.matches(env)) {
387 // ignore selector range
388 if (env.layer == null) {
389 env.layer = "default";
390 }
391 r.execute(env);
392 }
393 }
394 env.layer = "default";
395 return env;
396 }
397
398 /**
399 * Create image file from StyleElement.
400 *
401 * @return the URL
402 */
403 String createImage(StyleElement element, final String type, NavigatableComponent nc) {
404 BufferedImage img = new BufferedImage(16, 16, BufferedImage.TYPE_INT_ARGB);
405 Graphics2D g = img.createGraphics();
406 g.setClip(0, 0, 16, 16);
407 StyledMapRenderer renderer = new StyledMapRenderer(g, nc, false);
408 renderer.getSettings(false);
409 element.paintPrimitive(osm, MapPaintSettings.INSTANCE, renderer, false, false, false);
410 final String imageName = type + "_" + tag + ".png";
411 try (OutputStream out = Files.newOutputStream(options.imageDir.resolve(imageName))) {
412 ImageIO.write(img, "png", out);
413 } catch (IOException e) {
414 throw new UncheckedIOException(e);
415 }
416 final String baseUrl = options.imageUrlPrefix != null ? options.imageUrlPrefix : options.imageDir.toString();
417 return baseUrl + "/" + imageName;
418 }
419
420 /**
421 * Checks, if tag is supported and find URL for image icon in this case.
422 *
423 * @param generateImage if true, create or find a suitable image icon and return URL,
424 * if false, just check if tag is supported and return true or false
425 */
426 abstract Optional<String> findUrl(boolean generateImage);
427
428 protected Tag tag;
429 protected OsmPrimitive osm;
430 }
431
432 private class NodeChecker extends Checker {
433 NodeChecker(Tag tag) {
434 super(tag);
435 }
436
437 @Override
438 Optional<String> findUrl(boolean generateImage) {
439 this.osm = new Node(LatLon.ZERO);
440 Environment env = applyStylesheet(osm);
441 Cascade c = env.mc.getCascade("default");
442 Object image = c.get("icon-image");
443 if (image instanceof MapPaintStyles.IconReference && !((MapPaintStyles.IconReference) image).isDeprecatedIcon()) {
444 return Optional.of(options.findImageUrl(((MapPaintStyles.IconReference) image).iconName));
445 }
446 return Optional.empty();
447 }
448
449 }
450
451 private class WayChecker extends Checker {
452 WayChecker(Tag tag) {
453 super(tag);
454 }
455
456 @Override
457 Optional<String> findUrl(boolean generateImage) {
458 this.osm = new Way();
459 NavigatableComponent nc = new NavigatableComponent();
460 Node n1 = new Node(nc.getLatLon(2, 8));
461 Node n2 = new Node(nc.getLatLon(14, 8));
462 ((Way) osm).addNode(n1);
463 ((Way) osm).addNode(n2);
464 Environment env = applyStylesheet(osm);
465 LineElement les = LineElement.createLine(env);
466 if (les != null) {
467 if (!generateImage) return Optional.of("");
468 return Optional.of(createImage(les, "way", nc));
469 }
470 return Optional.empty();
471 }
472
473 }
474
475 private class AreaChecker extends Checker {
476 AreaChecker(Tag tag) {
477 super(tag);
478 }
479
480 @Override
481 Optional<String> findUrl(boolean generateImage) {
482 this.osm = new Way();
483 NavigatableComponent nc = new NavigatableComponent();
484 Node n1 = new Node(nc.getLatLon(2, 2));
485 Node n2 = new Node(nc.getLatLon(14, 2));
486 Node n3 = new Node(nc.getLatLon(14, 14));
487 Node n4 = new Node(nc.getLatLon(2, 14));
488 ((Way) osm).addNode(n1);
489 ((Way) osm).addNode(n2);
490 ((Way) osm).addNode(n3);
491 ((Way) osm).addNode(n4);
492 ((Way) osm).addNode(n1);
493 Environment env = applyStylesheet(osm);
494 AreaElement aes = AreaElement.create(env);
495 if (aes != null) {
496 if (!generateImage) return Optional.of("");
497 return Optional.of(createImage(aes, "area", nc));
498 }
499 return Optional.empty();
500 }
501 }
502 }
503
504 /**
505 * POJO representing a <a href="https://wiki.openstreetmap.org/wiki/Taginfo/Projects">Taginfo tag</a>.
506 */
507 private static class TagInfoTag {
508 final String description;
509 final String key;
510 final String value;
511 final Set<Type> objectTypes;
512 final String iconURL;
513
514 TagInfoTag(String description, String key, String value, Set<Type> objectTypes, String iconURL) {
515 this.description = description;
516 this.key = key;
517 this.value = value;
518 this.objectTypes = objectTypes;
519 this.iconURL = iconURL;
520 }
521
522 JsonObjectBuilder toJson() {
523 final JsonObjectBuilder object = Json.createObjectBuilder();
524 if (description != null) {
525 object.add("description", description);
526 }
527 object.add("key", key);
528 object.add("value", value);
529 if ((!objectTypes.isEmpty())) {
530 final JsonArrayBuilder types = Json.createArrayBuilder();
531 objectTypes.stream().map(Enum::name).map(String::toLowerCase).forEach(types::add);
532 object.add("object_types", types);
533 }
534 if (iconURL != null) {
535 object.add("icon_url", iconURL);
536 }
537 return object;
538 }
539
540 enum Type {
541 NODE, WAY, AREA, RELATION
542 }
543 }
544
545 /**
546 * Initialize the script.
547 */
548 private void init() throws IOException {
549 Logging.setLogLevel(Logging.LEVEL_INFO);
550 Preferences.main().enableSaveOnPut(false);
551 Config.setPreferencesInstance(Preferences.main());
552 Config.setBaseDirectoriesProvider(JosmBaseDirectories.getInstance());
553 Config.setUrlsProvider(JosmUrls.getInstance());
554 ProjectionRegistry.setProjection(Projections.getProjectionByCode("EPSG:3857"));
555 Path tmpdir = Files.createTempDirectory(options.baseDir, "pref");
556 tmpdir.toFile().deleteOnExit();
557 System.setProperty("josm.home", tmpdir.toString());
558 DeleteCommand.setDeletionCallback(DeleteAction.defaultDeletionCallback);
559 Territories.initialize();
560 RightAndLefthandTraffic.initialize();
561 Files.createDirectories(options.imageDir);
562 }
563}
Note: See TracBrowser for help on using the repository browser.