Merge pull request #10 from rosm-project/prepare-publication

Prepare publication

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continuous integration
+name: continuous-integration
 
 on: [push, pull_request]
 
@@ -13,30 +13,18 @@ jobs:
           - --no-default-features
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Install toolchain
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
-        profile: minimal
-        toolchain: stable
-        override: true
-    - name: Cargo format
-      uses: actions-rs/cargo@v1
-      with:
-        command: fmt
-        args: --check
-    - name: Cargo check
-      uses: actions-rs/cargo@v1
-      with:
-        command: check
-        args: ${{ matrix.features }}
-    - name: Cargo build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: ${{ matrix.features }}
-    - name: Cargo test
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: ${{ matrix.features }}
+        components: rustfmt
+    - name: Install protoc
+      run: sudo apt-get install protobuf-compiler
+    - name: Check formatting
+      run: cargo fmt --all -- --check
+    - name: Check
+      run: cargo check ${{ matrix.features }}
+    - name: Build
+      run: cargo build ${{ matrix.features }}
+    - name: Test
+      run: cargo test ${{ matrix.features }}

Cargo.toml

@@ -1,18 +1,25 @@
 [package]
 name = "rosm_pbf_reader"
-version = "0.4.0"
+version = "1.0.0"
 authors = ["Zsolt Bölöny <[email protected]>"]
 edition = "2021"
+license = "MIT"
+description = "A low-level Rust library for parsing OpenStreetMap data in PBF format"
+repository = "https://github.com/rosm-project/rosm_pbf_reader"
+keywords = ["osm", "openstreetmap", "osmpbf"]
+categories = ["parser-implementations", "encoding"]
 
 [dependencies]
-flate2 = { version = "1.0", optional = true }
-prost = "0.10"
+flate2 = { version = "1.0.25", optional = true }
+prost = "0.11.6"
 
 [features]
 default = ["flate2"]
 
 [build-dependencies]
-prost-build = "0.10"
+prost-build = "0.11.6"
 
 [dev-dependencies]
-threadpool = "1.8"
+env_logger = "0.10.0"
+log = "0.4.17"
+threadpool = "1.8.1"

README.md

@@ -1,10 +1,18 @@
 # rosm_pbf_reader
 
+[![Crates.io](https://img.shields.io/crates/v/rosm_pbf_reader.svg?label=rosm_pbf_reader)](https://crates.io/crates/rosm_pbf_reader)
+[![Docs.rs](https://docs.rs/rosm_pbf_reader/badge.svg)](https://docs.rs/rosm_pbf_reader)
+[![Build Status](https://github.com/yzsolt/rosm_pbf_reader/workflows/continuous-integration/badge.svg)](https://github.com/yzsolt/rosm_pbf_reader/actions)
+
 A low-level Rust library for parsing OpenStreetMap data in [PBF format](https://wiki.openstreetmap.org/wiki/PBF_Format).
 
 This library provides the smallest possible API to work with OSM PBF files: a blob reader, a block parser and some utilities to read delta or densely encoded data. No other utilities are provided for further data processing (like filtering). There's also no built-in parallelization, however block parsing (which is the most computation-heavy part of the process) can be easily dispatched to multiple threads.
 
-The library uses [quick-protobuf](https://github.com/tafia/quick-protobuf) for fast protobuf parsing with minimal allocations.
+## Features
+
+Since most OSM PBFs are ZLib compressed, ZLib decompression support using [`flate2`](https://crates.io/crates/flate2) is enabled by default. See Cargo's [default feature documentation](https://doc.rust-lang.org/cargo/reference/features.html#the-default-feature) how to disable it.
+
+The library provides a way for the user to support other compression methods by implementing the `Decompressor` trait.
 
 ## Examples
 

examples/count_wikidata.rs

@@ -1,5 +1,7 @@
-use rosm_pbf_reader::pbf;
-use rosm_pbf_reader::{read_blob, Block, BlockParser, DenseNodeReader, TagReader};
+use log::{error, info, warn};
+
+use rosm_pbf_reader::dense::{new_dense_tag_reader, DenseNodeReader};
+use rosm_pbf_reader::{new_tag_reader, pbf, read_blob, Block, BlockParser, Error, RawBlock};
 
 use std::cell::RefCell;
 use std::fs::File;
@@ -11,7 +13,7 @@ static WIKIDATA_COUNT: AtomicUsize = AtomicUsize::new(0);
 
 fn process_header_block(block: pbf::HeaderBlock) {
     if let Some(writing_program) = &block.writingprogram {
-        println!("Writing program: {}", writing_program);
+        info!("Writing program: {}", writing_program);
     }
 }
 
@@ -21,30 +23,56 @@ fn process_tag(key: &str, _value: &str) {
     }
 }
 
-fn process_primitive_block(block: pbf::PrimitiveBlock) {
+fn process_primitive_block(block: pbf::PrimitiveBlock) -> Result<(), Error> {
     for group in &block.primitivegroup {
         let string_table = &block.stringtable;
 
         for way in &group.ways {
-            let tags = TagReader::new(&way.keys, &way.vals, string_table);
+            let tags = new_tag_reader(string_table, &way.keys, &way.vals);
             for (key, value) in tags {
                 process_tag(key.unwrap(), value.unwrap());
             }
         }
 
         if let Some(dense_nodes) = &group.dense {
-            let nodes = DenseNodeReader::new(&dense_nodes, string_table);
+            let nodes = DenseNodeReader::new(&dense_nodes)?;
 
             for node in nodes {
-                for (key, value) in node.tags {
+                let tags = new_dense_tag_reader(string_table, node?.key_value_indices);
+
+                for (key, value) in tags {
                     process_tag(key.unwrap(), value.unwrap());
                 }
             }
         }
     }
+
+    Ok(())
+}
+
+fn parse_block(block_parser: &mut BlockParser, raw_block: RawBlock) {
+    match block_parser.parse_block(raw_block) {
+        Ok(block) => match block {
+            Block::Header(header_block) => process_header_block(header_block),
+            Block::Primitive(primitive_block) => match process_primitive_block(primitive_block) {
+                Err(error) => {
+                    error!("Error during processing a primitive block: {:?}", error)
+                }
+                _ => {}
+            },
+            Block::Unknown(unknown_block) => {
+                warn!("Skipping unknown block of size {}", unknown_block.len())
+            }
+        },
+        Err(error) => error!("Error during parsing a block: {:?}", error),
+    }
 }
 
 fn main() {
+    let mut builder = env_logger::Builder::from_default_env();
+    builder.filter_level(log::LevelFilter::Info);
+    builder.init();
+
     let mut args = std::env::args();
 
     let pbf_path = args.nth(1).expect("Expected an OSM PBF file as first argument");
@@ -63,17 +91,8 @@ fn main() {
 
         while let Some(result) = read_blob(&mut file) {
             match result {
-                Ok(raw_block) => match block_parser.parse_block(raw_block) {
-                    Ok(block) => match block {
-                        Block::Header(header_block) => process_header_block(header_block),
-                        Block::Primitive(primitive_block) => process_primitive_block(primitive_block),
-                        Block::Unknown(unknown_block) => {
-                            println!("Skipping unknown block of size {}", unknown_block.len())
-                        }
-                    },
-                    Err(error) => println!("Error during parsing a block: {:?}", error),
-                },
-                Err(error) => println!("Error during reading the next blob: {:?}", error),
+                Ok(raw_block) => parse_block(&mut block_parser, raw_block),
+                Err(error) => error!("Error during reading the next blob: {:?}", error),
             }
         }
     } else {
@@ -84,33 +103,23 @@ fn main() {
 
         while let Some(result) = read_blob(&mut file) {
             match result {
-                Ok(blob) => {
+                Ok(raw_block) => {
                     thread_pool.execute(move || {
                         BLOCK_PARSER.with(|block_parser| {
                             let mut block_parser = block_parser.borrow_mut();
-
-                            match block_parser.parse_block(blob) {
-                                Ok(block) => match block {
-                                    Block::Header(header_block) => process_header_block(header_block),
-                                    Block::Primitive(primitive_block) => process_primitive_block(primitive_block),
-                                    Block::Unknown(unknown_block) => {
-                                        println!("Skipping unknown block of size {}", unknown_block.len())
-                                    }
-                                },
-                                Err(error) => println!("Error during parsing a block: {:?}", error),
-                            }
+                            parse_block(&mut block_parser, raw_block);
                         });
                     });
                 }
-                Err(error) => println!("Error during reading the next blob: {:?}", error),
+                Err(error) => error!("Error during reading the next blob: {:?}", error),
             }
         }
 
         thread_pool.join();
     }
 
-    println!("Wikidata tag count: {}", WIKIDATA_COUNT.load(Ordering::SeqCst));
-    println!(
+    info!("Wikidata tag count: {}", WIKIDATA_COUNT.load(Ordering::SeqCst));
+    info!(
         "Finished in {:.2}s on {} thread(s)",
         start.elapsed().as_secs_f64(),
         thread_count