001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.io.InputStreamReader; 009import java.text.MessageFormat; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.List; 013import java.util.Objects; 014import java.util.regex.Matcher; 015import java.util.regex.Pattern; 016 017import javax.xml.stream.Location; 018import javax.xml.stream.XMLInputFactory; 019import javax.xml.stream.XMLStreamConstants; 020import javax.xml.stream.XMLStreamException; 021import javax.xml.stream.XMLStreamReader; 022 023import org.openstreetmap.josm.Main; 024import org.openstreetmap.josm.data.Bounds; 025import org.openstreetmap.josm.data.DataSource; 026import org.openstreetmap.josm.data.coor.LatLon; 027import org.openstreetmap.josm.data.osm.Changeset; 028import org.openstreetmap.josm.data.osm.DataSet; 029import org.openstreetmap.josm.data.osm.Node; 030import org.openstreetmap.josm.data.osm.NodeData; 031import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 032import org.openstreetmap.josm.data.osm.PrimitiveData; 033import org.openstreetmap.josm.data.osm.Relation; 034import org.openstreetmap.josm.data.osm.RelationData; 035import org.openstreetmap.josm.data.osm.RelationMemberData; 036import org.openstreetmap.josm.data.osm.Tagged; 037import org.openstreetmap.josm.data.osm.User; 038import org.openstreetmap.josm.data.osm.Way; 039import org.openstreetmap.josm.data.osm.WayData; 040import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 041import org.openstreetmap.josm.gui.progress.ProgressMonitor; 042import org.openstreetmap.josm.tools.CheckParameterUtil; 043import org.openstreetmap.josm.tools.date.DateUtils; 044 045/** 046 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 047 * 048 * For each xml element, there is a dedicated method. 049 * The XMLStreamReader cursor points to the start of the element, when the method is 050 * entered, and it must point to the end of the same element, when it is exited. 051 */ 052public class OsmReader extends AbstractReader { 053 054 protected XMLStreamReader parser; 055 056 protected boolean cancel; 057 058 /** Used by plugins to register themselves as data postprocessors. */ 059 private static volatile List<OsmServerReadPostprocessor> postprocessors; 060 061 /** Register a new postprocessor. 062 * @param pp postprocessor 063 * @see #deregisterPostprocessor 064 */ 065 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 066 if (postprocessors == null) { 067 postprocessors = new ArrayList<>(); 068 } 069 postprocessors.add(pp); 070 } 071 072 /** 073 * Deregister a postprocessor previously registered with {@link #registerPostprocessor}. 074 * @param pp postprocessor 075 * @see #registerPostprocessor 076 */ 077 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 078 if (postprocessors != null) { 079 postprocessors.remove(pp); 080 } 081 } 082 083 /** 084 * constructor (for private and subclasses use only) 085 * 086 * @see #parseDataSet(InputStream, ProgressMonitor) 087 */ 088 protected OsmReader() { 089 // Restricts visibility 090 } 091 092 protected void setParser(XMLStreamReader parser) { 093 this.parser = parser; 094 } 095 096 protected void throwException(String msg, Throwable th) throws XMLStreamException { 097 throw new XmlStreamParsingException(msg, parser.getLocation(), th); 098 } 099 100 protected void throwException(String msg) throws XMLStreamException { 101 throw new XmlStreamParsingException(msg, parser.getLocation()); 102 } 103 104 protected void parse() throws XMLStreamException { 105 int event = parser.getEventType(); 106 while (true) { 107 if (event == XMLStreamConstants.START_ELEMENT) { 108 parseRoot(); 109 } else if (event == XMLStreamConstants.END_ELEMENT) 110 return; 111 if (parser.hasNext()) { 112 event = parser.next(); 113 } else { 114 break; 115 } 116 } 117 parser.close(); 118 } 119 120 protected void parseRoot() throws XMLStreamException { 121 if ("osm".equals(parser.getLocalName())) { 122 parseOsm(); 123 } else { 124 parseUnknown(); 125 } 126 } 127 128 private void parseOsm() throws XMLStreamException { 129 String v = parser.getAttributeValue(null, "version"); 130 if (v == null) { 131 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 132 } 133 if (!"0.6".equals(v)) { 134 throwException(tr("Unsupported version: {0}", v)); 135 } 136 ds.setVersion(v); 137 String upload = parser.getAttributeValue(null, "upload"); 138 if (upload != null) { 139 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 140 } 141 String generator = parser.getAttributeValue(null, "generator"); 142 Long uploadChangesetId = null; 143 if (parser.getAttributeValue(null, "upload-changeset") != null) { 144 uploadChangesetId = getLong("upload-changeset"); 145 } 146 while (true) { 147 int event = parser.next(); 148 149 if (cancel) { 150 cancel = false; 151 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 152 } 153 154 if (event == XMLStreamConstants.START_ELEMENT) { 155 switch (parser.getLocalName()) { 156 case "bounds": 157 parseBounds(generator); 158 break; 159 case "node": 160 parseNode(); 161 break; 162 case "way": 163 parseWay(); 164 break; 165 case "relation": 166 parseRelation(); 167 break; 168 case "changeset": 169 parseChangeset(uploadChangesetId); 170 break; 171 default: 172 parseUnknown(); 173 } 174 } else if (event == XMLStreamConstants.END_ELEMENT) 175 return; 176 } 177 } 178 179 private void parseBounds(String generator) throws XMLStreamException { 180 String minlon = parser.getAttributeValue(null, "minlon"); 181 String minlat = parser.getAttributeValue(null, "minlat"); 182 String maxlon = parser.getAttributeValue(null, "maxlon"); 183 String maxlat = parser.getAttributeValue(null, "maxlat"); 184 String origin = parser.getAttributeValue(null, "origin"); 185 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 186 if (origin == null) { 187 origin = generator; 188 } 189 Bounds bounds = new Bounds( 190 Double.parseDouble(minlat), Double.parseDouble(minlon), 191 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 192 if (bounds.isOutOfTheWorld()) { 193 Bounds copy = new Bounds(bounds); 194 bounds.normalize(); 195 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds); 196 } 197 DataSource src = new DataSource(bounds, origin); 198 ds.dataSources.add(src); 199 } else { 200 throwException(tr("Missing mandatory attributes on element ''bounds''. " + 201 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 202 minlon, minlat, maxlon, maxlat, origin 203 )); 204 } 205 jumpToEnd(); 206 } 207 208 protected Node parseNode() throws XMLStreamException { 209 NodeData nd = new NodeData(); 210 String lat = parser.getAttributeValue(null, "lat"); 211 String lon = parser.getAttributeValue(null, "lon"); 212 if (lat != null && lon != null) { 213 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 214 } 215 readCommon(nd); 216 Node n = new Node(nd.getId(), nd.getVersion()); 217 n.setVisible(nd.isVisible()); 218 n.load(nd); 219 externalIdMap.put(nd.getPrimitiveId(), n); 220 while (true) { 221 int event = parser.next(); 222 if (event == XMLStreamConstants.START_ELEMENT) { 223 if ("tag".equals(parser.getLocalName())) { 224 parseTag(n); 225 } else { 226 parseUnknown(); 227 } 228 } else if (event == XMLStreamConstants.END_ELEMENT) 229 return n; 230 } 231 } 232 233 protected Way parseWay() throws XMLStreamException { 234 WayData wd = new WayData(); 235 readCommon(wd); 236 Way w = new Way(wd.getId(), wd.getVersion()); 237 w.setVisible(wd.isVisible()); 238 w.load(wd); 239 externalIdMap.put(wd.getPrimitiveId(), w); 240 241 Collection<Long> nodeIds = new ArrayList<>(); 242 while (true) { 243 int event = parser.next(); 244 if (event == XMLStreamConstants.START_ELEMENT) { 245 switch (parser.getLocalName()) { 246 case "nd": 247 nodeIds.add(parseWayNode(w)); 248 break; 249 case "tag": 250 parseTag(w); 251 break; 252 default: 253 parseUnknown(); 254 } 255 } else if (event == XMLStreamConstants.END_ELEMENT) { 256 break; 257 } 258 } 259 if (w.isDeleted() && !nodeIds.isEmpty()) { 260 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId())); 261 nodeIds = new ArrayList<>(); 262 } 263 ways.put(wd.getUniqueId(), nodeIds); 264 return w; 265 } 266 267 private long parseWayNode(Way w) throws XMLStreamException { 268 if (parser.getAttributeValue(null, "ref") == null) { 269 throwException( 270 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 271 ); 272 } 273 long id = getLong("ref"); 274 if (id == 0) { 275 throwException( 276 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 277 ); 278 } 279 jumpToEnd(); 280 return id; 281 } 282 283 protected Relation parseRelation() throws XMLStreamException { 284 RelationData rd = new RelationData(); 285 readCommon(rd); 286 Relation r = new Relation(rd.getId(), rd.getVersion()); 287 r.setVisible(rd.isVisible()); 288 r.load(rd); 289 externalIdMap.put(rd.getPrimitiveId(), r); 290 291 Collection<RelationMemberData> members = new ArrayList<>(); 292 while (true) { 293 int event = parser.next(); 294 if (event == XMLStreamConstants.START_ELEMENT) { 295 switch (parser.getLocalName()) { 296 case "member": 297 members.add(parseRelationMember(r)); 298 break; 299 case "tag": 300 parseTag(r); 301 break; 302 default: 303 parseUnknown(); 304 } 305 } else if (event == XMLStreamConstants.END_ELEMENT) { 306 break; 307 } 308 } 309 if (r.isDeleted() && !members.isEmpty()) { 310 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId())); 311 members = new ArrayList<>(); 312 } 313 relations.put(rd.getUniqueId(), members); 314 return r; 315 } 316 317 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 318 OsmPrimitiveType type = null; 319 long id = 0; 320 String value = parser.getAttributeValue(null, "ref"); 321 if (value == null) { 322 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId())); 323 } 324 try { 325 id = Long.parseLong(value); 326 } catch (NumberFormatException e) { 327 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()), 328 value), e); 329 } 330 value = parser.getAttributeValue(null, "type"); 331 if (value == null) { 332 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 333 } 334 try { 335 type = OsmPrimitiveType.fromApiTypeName(value); 336 } catch (IllegalArgumentException e) { 337 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", 338 Long.toString(id), Long.toString(r.getUniqueId()), value), e); 339 } 340 String role = parser.getAttributeValue(null, "role"); 341 342 if (id == 0) { 343 throwException(tr("Incomplete <member> specification with ref=0")); 344 } 345 jumpToEnd(); 346 return new RelationMemberData(role, type, id); 347 } 348 349 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 350 351 Long id = null; 352 if (parser.getAttributeValue(null, "id") != null) { 353 id = getLong("id"); 354 } 355 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 356 if (Objects.equals(id, uploadChangesetId)) { 357 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 358 while (true) { 359 int event = parser.next(); 360 if (event == XMLStreamConstants.START_ELEMENT) { 361 if ("tag".equals(parser.getLocalName())) { 362 parseTag(uploadChangeset); 363 } else { 364 parseUnknown(); 365 } 366 } else if (event == XMLStreamConstants.END_ELEMENT) 367 return; 368 } 369 } else { 370 jumpToEnd(false); 371 } 372 } 373 374 private void parseTag(Tagged t) throws XMLStreamException { 375 String key = parser.getAttributeValue(null, "k"); 376 String value = parser.getAttributeValue(null, "v"); 377 if (key == null || value == null) { 378 throwException(tr("Missing key or value attribute in tag.")); 379 } else { 380 t.put(key.intern(), value.intern()); 381 } 382 jumpToEnd(); 383 } 384 385 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 386 final String element = parser.getLocalName(); 387 if (printWarning && ("note".equals(element) || "meta".equals(element))) { 388 // we know that Overpass API returns those elements 389 Main.debug(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 390 } else if (printWarning) { 391 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", element)); 392 } 393 while (true) { 394 int event = parser.next(); 395 if (event == XMLStreamConstants.START_ELEMENT) { 396 parseUnknown(false); /* no more warning for inner elements */ 397 } else if (event == XMLStreamConstants.END_ELEMENT) 398 return; 399 } 400 } 401 402 protected void parseUnknown() throws XMLStreamException { 403 parseUnknown(true); 404 } 405 406 /** 407 * When cursor is at the start of an element, moves it to the end tag of that element. 408 * Nested content is skipped. 409 * 410 * This is basically the same code as parseUnknown(), except for the warnings, which 411 * are displayed for inner elements and not at top level. 412 * @param printWarning if {@code true}, a warning message will be printed if an unknown element is met 413 * @throws XMLStreamException if there is an error processing the underlying XML source 414 */ 415 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 416 while (true) { 417 int event = parser.next(); 418 if (event == XMLStreamConstants.START_ELEMENT) { 419 parseUnknown(printWarning); 420 } else if (event == XMLStreamConstants.END_ELEMENT) 421 return; 422 } 423 } 424 425 private void jumpToEnd() throws XMLStreamException { 426 jumpToEnd(true); 427 } 428 429 private User createUser(String uid, String name) throws XMLStreamException { 430 if (uid == null) { 431 if (name == null) 432 return null; 433 return User.createLocalUser(name); 434 } 435 try { 436 long id = Long.parseLong(uid); 437 return User.createOsmUser(id, name); 438 } catch (NumberFormatException e) { 439 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); 440 } 441 return null; 442 } 443 444 /** 445 * Read out the common attributes and put them into current OsmPrimitive. 446 * @param current primitive to update 447 * @throws XMLStreamException if there is an error processing the underlying XML source 448 */ 449 private void readCommon(PrimitiveData current) throws XMLStreamException { 450 current.setId(getLong("id")); 451 if (current.getUniqueId() == 0) { 452 throwException(tr("Illegal object with ID=0.")); 453 } 454 455 String time = parser.getAttributeValue(null, "timestamp"); 456 if (time != null && !time.isEmpty()) { 457 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000)); 458 } 459 460 String user = parser.getAttributeValue(null, "user"); 461 String uid = parser.getAttributeValue(null, "uid"); 462 current.setUser(createUser(uid, user)); 463 464 String visible = parser.getAttributeValue(null, "visible"); 465 if (visible != null) { 466 current.setVisible(Boolean.parseBoolean(visible)); 467 } 468 469 String versionString = parser.getAttributeValue(null, "version"); 470 int version = 0; 471 if (versionString != null) { 472 try { 473 version = Integer.parseInt(versionString); 474 } catch (NumberFormatException e) { 475 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 476 Long.toString(current.getUniqueId()), versionString), e); 477 } 478 switch (ds.getVersion()) { 479 case "0.6": 480 if (version <= 0 && !current.isNew()) { 481 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 482 Long.toString(current.getUniqueId()), versionString)); 483 } else if (version < 0 && current.isNew()) { 484 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", 485 current.getUniqueId(), version, 0, "0.6")); 486 version = 0; 487 } 488 break; 489 default: 490 // should not happen. API version has been checked before 491 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 492 } 493 } else { 494 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 495 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { 496 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 497 } 498 } 499 current.setVersion(version); 500 501 String action = parser.getAttributeValue(null, "action"); 502 if (action == null) { 503 // do nothing 504 } else if ("delete".equals(action)) { 505 current.setDeleted(true); 506 current.setModified(current.isVisible()); 507 } else if ("modify".equals(action)) { 508 current.setModified(true); 509 } 510 511 String v = parser.getAttributeValue(null, "changeset"); 512 if (v == null) { 513 current.setChangesetId(0); 514 } else { 515 try { 516 current.setChangesetId(Integer.parseInt(v)); 517 } catch (IllegalArgumentException e) { 518 Main.debug(e.getMessage()); 519 if (current.isNew()) { 520 // for a new primitive we just log a warning 521 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 522 v, current.getUniqueId())); 523 current.setChangesetId(0); 524 } else { 525 // for an existing primitive this is a problem 526 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); 527 } 528 } catch (IllegalStateException e) { 529 // thrown for positive changeset id on new primitives 530 Main.debug(e); 531 Main.info(e.getMessage()); 532 current.setChangesetId(0); 533 } 534 if (current.getChangesetId() <= 0) { 535 if (current.isNew()) { 536 // for a new primitive we just log a warning 537 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 538 v, current.getUniqueId())); 539 current.setChangesetId(0); 540 } else { 541 // for an existing primitive this is a problem 542 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 543 } 544 } 545 } 546 } 547 548 private long getLong(String name) throws XMLStreamException { 549 String value = parser.getAttributeValue(null, name); 550 if (value == null) { 551 throwException(tr("Missing required attribute ''{0}''.", name)); 552 } 553 try { 554 return Long.parseLong(value); 555 } catch (NumberFormatException e) { 556 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e); 557 } 558 return 0; // should not happen 559 } 560 561 /** 562 * Exception thrown after user cancelation. 563 */ 564 private static final class OsmParsingCanceledException extends XmlStreamParsingException implements ImportCancelException { 565 /** 566 * Constructs a new {@code OsmParsingCanceledException}. 567 * @param msg The error message 568 * @param location The parser location 569 */ 570 OsmParsingCanceledException(String msg, Location location) { 571 super(msg, location); 572 } 573 } 574 575 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 576 if (progressMonitor == null) { 577 progressMonitor = NullProgressMonitor.INSTANCE; 578 } 579 ProgressMonitor.CancelListener cancelListener = () -> cancel = true; 580 progressMonitor.addCancelListener(cancelListener); 581 CheckParameterUtil.ensureParameterNotNull(source, "source"); 582 try { 583 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 584 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 585 586 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 587 XMLInputFactory factory = XMLInputFactory.newInstance(); 588 // do not try to load external entities 589 factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); 590 factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); 591 setParser(factory.createXMLStreamReader(ir)); 592 parse(); 593 } 594 progressMonitor.worked(1); 595 596 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 597 prepareDataSet(); 598 progressMonitor.worked(1); 599 600 // iterate over registered postprocessors and give them each a chance 601 // to modify the dataset we have just loaded. 602 if (postprocessors != null) { 603 for (OsmServerReadPostprocessor pp : postprocessors) { 604 pp.postprocessDataSet(getDataSet(), progressMonitor); 605 } 606 } 607 return getDataSet(); 608 } catch (IllegalDataException e) { 609 throw e; 610 } catch (XmlStreamParsingException e) { 611 throw new IllegalDataException(e.getMessage(), e); 612 } catch (XMLStreamException e) { 613 String msg = e.getMessage(); 614 Pattern p = Pattern.compile("Message: (.+)"); 615 Matcher m = p.matcher(msg); 616 if (m.find()) { 617 msg = m.group(1); 618 } 619 if (e.getLocation() != null) 620 throw new IllegalDataException(tr("Line {0} column {1}: ", 621 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 622 else 623 throw new IllegalDataException(msg, e); 624 } catch (IOException e) { 625 throw new IllegalDataException(e); 626 } finally { 627 progressMonitor.finishTask(); 628 progressMonitor.removeCancelListener(cancelListener); 629 } 630 } 631 632 /** 633 * Parse the given input source and return the dataset. 634 * 635 * @param source the source input stream. Must not be null. 636 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 637 * 638 * @return the dataset with the parsed data 639 * @throws IllegalDataException if an error was found while parsing the data from the source 640 * @throws IllegalArgumentException if source is null 641 */ 642 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 643 return new OsmReader().doParseDataSet(source, progressMonitor); 644 } 645}