Changeset 28983 in osm for applications/editors

Timestamp:

2012-11-25T01:08:09+01:00 (12 years ago)

Author:

simon04

Message:

JOSM/wikipedia: migrate to CatScan2, Unicode handling (fix #josm7913), category depth to prefs

Location:

applications/editors/josm/plugins/wikipedia/src/org/wikipedia

Files:

• WikipediaApp.java (modified) (2 diffs)
• WikipediaToggleDialog.java (modified) (1 diff)

applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java

@@ -65 +65 @@
     }
 
-    static List<WikipediaEntry> getEntriesFromCategory(String wikipediaLang, String category) {
+    static List<WikipediaEntry> getEntriesFromCategory(String wikipediaLang, String category, int depth) {
         try {
-            final String url = "http://toolserver.org/~daniel/WikiSense/CategoryIntersect.php?"
-                    + "wikilang=" + wikipediaLang
-                    + "&wikifam=.wikipedia.org"
-                    + "&basecat=" + encodeURL(category)
-                    + "&basedeep=3&templates=&mode=al&format=csv";
+            final String url = "http://toolserver.org/~magnus/catscan_rewrite.php"
+                    + "?project=wikipedia"
+                    + "&ext_image_data="
+                    + "&file_usage_data="
+                    + "&show_redirects=no"
+                    + "&format=tsv"
+                    + "&doit=doit"
+                    + "&depth=" + depth
+                    + "&language=" + wikipediaLang
+                    + "&categories=" + encodeURL(category);
             System.out.println("Wikipedia: GET " + url);
-            final Scanner scanner = new Scanner(new URL(url).openStream()).useDelimiter("\n");
+            final Scanner scanner = new Scanner(new URL(url).openStream(), "UTF-8").useDelimiter("\n");
             final List<WikipediaEntry> entries = new ArrayList<WikipediaEntry>();
+            {
+                // skip first two rows
+                final String header = scanner.next(); // |subset
+                if (!"|subset".equals(header.trim())) {
+                    throw new RuntimeException("Wikipedia: Invalid header: " + header);
+                }
+                final String colspec = scanner.next(); //title  id      namespace       len     touched seen    nstext
+                if (!colspec.startsWith("title")) {
+                    throw new RuntimeException("Wikipedia: Invalid colspec: " + colspec);
+                }
+            }
             while (scanner.hasNext()) {
-                // TODO ignore redirects
-                final String[] x = scanner.next().split("\t");
-                if ("0".equals(x[0].trim())) { // denotes article
-                    final String article = x[1].replace("_", " ");
-                    entries.add(new WikipediaEntry(article, wikipediaLang, article));
-                }
+                final String article = scanner.next().split("\t")[0].trim().replace("_", " ");
+                entries.add(new WikipediaEntry(article, wikipediaLang, article));
             }
             return entries;
@@ -96 +108 @@
         Map<String, Boolean> status = new HashMap<String, Boolean>();
         if (!articleNames.isEmpty()) {
-            final String url = "http://toolserver.org/~simon04/getGeoJSON.php?action=check"
+            final String url = "http://toolserver.org/~master/osmjson/getGeoJSON.php?action=check"
                     + "&lang=" + wikipediaLang;
             System.out.println("Wikipedia: GET " + url);

applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaToggleDialog.java

@@ -145 +145 @@
                 return;
             }
-            List<WikipediaEntry> entries = WikipediaApp.getEntriesFromCategory(wikipediaLang.get(), category);
+            List<WikipediaEntry> entries = WikipediaApp.getEntriesFromCategory(
+                    wikipediaLang.get(), category, Main.pref.getInteger("wikipedia.depth", 3));
             setWikipediaEntries(entries);
             setTitle(/* I18n: [language].Wikipedia.org: [category] */ tr("{0}.Wikipedia.org: {1}", wikipediaLang.get(), category));