add mdbook

.dockerignore

@@ -2,3 +2,4 @@
 .env
 venv
 target
+/book

.github/workflows/book.yml

@@ -0,0 +1,70 @@
+name: book
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'book/**'
+      - 'book.toml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'book/**'
+      - 'book.toml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: test
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install mdbook
+        run: |
+          mkdir mdbook
+          curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.14/mdbook-v0.4.14-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook
+          echo `pwd`/mdbook >> $GITHUB_PATH
+
+      - name: Run tests
+        run: mdbook test
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install mdbook
+        run: |
+          mkdir mdbook
+          curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.14/mdbook-v0.4.14-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook
+          echo `pwd`/mdbook >> $GITHUB_PATH
+
+      - name: Build
+        run: mdbook build
+
+      - name: Save pages artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: target/book
+
+  deploy:
+    # Only deploy if a push to main
+    if: github.ref_name == 'main' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [test, build]
+
+    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
+    permissions:
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v1

README.md

@@ -12,7 +12,7 @@ Trin currently only runs on Unix-based platforms (Linux, macOS). We plan to even
 
 ## How to use Trin
 
-Check out the [Getting Started](/docs/getting_started.md) guide to quickly get up and running with Trin.
+Check out the [Trin book](https://ethereum.github.io/trin) to quickly get up and running with Trin.
 
 ## Experimental Status
 
@@ -23,7 +23,7 @@ In this stage of development, Trin relies on a separate execution node (local no
 ## Want to help?
 
 Want to file a bug, contribute some code, or improve documentation? Excellent! Read up on our
-guidelines for [contributing](/docs/contributing.md),
+guidelines for contributing in the [Trin book](https://ethereum.github.io/trin),
 then check out issues that are labeled
 [Good First Issue](https://github.com/ethereum/trin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22).
 

book/.gitignore

@@ -0,0 +1,4 @@
+book
+mermaid.min.js
+mermaid-init.js
+

book/README.md

@@ -0,0 +1,22 @@
+## Using the book
+
+The book can be built and served locally.
+```sh
+cargo install mdbook
+```
+Install support for `mermaid` diagrams:
+```sh
+cd book
+cargo install mdbook-mermaid
+mdbook-mermaid install
+```
+This will create `mermaid.min.js` and `mermaid-init.js` files.
+
+Then run the book from the book crate:
+```sh
+mdbook serve --open
+```
+Or the project root:
+```sh
+mdbook serve --open ./book
+```

book/book.toml

@@ -0,0 +1,17 @@
+[book]
+language = "en"
+multilingual = false
+src = "src"
+title = "Trin"
+
+[output.html]
+additional-js = ["mermaid.min.js", "mermaid-init.js"]
+
+[output.html.fold]
+enable = true
+
+[preprocessor]
+
+[preprocessor.mermaid]
+command = "mdbook-mermaid"
+

book/src/SUMMARY.md

@@ -0,0 +1,64 @@
+# Summary
+
+- [Introduction](introduction/README.md)
+    - [Portal Network](introduction/portal_network.md)
+- [Users](users/README.md)
+    - [Requirements](users/requirements.md)
+    - [Installation](users/installation.md)
+        - [Mac Os](users/installation/mac_os.md)
+        - [Linux](users/installation/linux.md)
+        - [Raspberry Pi](users/installation/raspberry_pi.md)
+        - [Windows](users/installation/windows.md)
+    - [Startup](users/startup.md)
+    - [Use](users/use/README.md)
+        - [Making queries](users/use/making_queries.md)
+        - [Ethereum data](users/use/ethereum_data.md)
+        - [Portal network data](users/use/portal_network_data.md)
+    - [Monitoring](users/monitoring.md)
+    - [Problems](users/problems.md)
+    - [FAQ](users/faq.md)
+- [Developers](developers/README.md)
+    - [Quick setup](developers/quick_setup.md)
+    - [Developer stories](developers/developer_stories.md)
+    - [Goals](developers/goals.md)
+    - [Progress status](developers/progress_status.md)
+    - [Architecture](developers/architecture/README.md)
+        - [Workspaces](developers/architecture/workspaces.md)
+        - [Process flow](developers/architecture/process_flow.md)
+        - [Database](developers/architecture/database.md)
+        - [Testing](developers/architecture/testing.md)
+    - [Protocols](developers/protocols/README.md)
+        - [Portal wire protocol](developers/protocols/portal_wire.md)
+        - [Discovery](developers/protocols/discovery.md)
+        - [uTP](developers/protocols/utp.md)
+        - [JSON-RPC](developers/protocols/json_rpc.md)
+        - [SSZ](developers/protocols/ssz.md)
+        - [Kademlia](developers/protocols/kademlia.md)
+    - [Core concepts](developers/core_concepts/README.md)
+        - [Finding peers](developers/core_concepts/finding_peers.md)
+        - [Chain tip](developers/core_concepts/chain_tip.md)
+        - [Cryptographic accumulator](developers/core_concepts/cryptographic_accumulator.md)
+        - [Bridge](developers/core_concepts/bridge.md)
+        - [Archive nodes](developers/core_concepts/archive_nodes.md)
+    - [Contributor guidelines](developers/contributing/README.md)
+        - [Rust](developers/contributing/rust/README.md)
+            - [Comments](developers/contributing/rust/comments.md)
+            - [Imports](developers/contributing/rust/imports.md)
+            - [Logging](developers/contributing/rust/logging.md)
+            - [Error handling](developers/contributing/rust/error_handling.md)
+            - [Style](developers/contributing/rust/style.md)
+        - [Git](developers/contributing/git/README.md)
+            - [Commits](developers/contributing/git/commits.md)
+            - [Rebasing](developers/contributing/git/rebasing.md)
+            - [Release notes](developers/contributing/git/release_notes.md)
+            - [Pull requests](developers/contributing/git/pull_requests.md)
+            - [Code review](developers/contributing/git/code_review.md)
+            - [Fetching a pull request](developers/contributing/git/fetching_pull_requests.md)
+            - [Merging](developers/contributing/git/merging.md)
+        - [Releases](developers/contributing/releases/README.md)
+            - [Release notes](developers/contributing/releases/release_notes.md)
+            - [Versioning](developers/contributing/releases/versioning.md)
+            - [Generation](developers/contributing/releases/generation.md)
+        - [Tests](developers/contributing/tests.md)
+        - [Book](developers/contributing/book.md)
+

book/src/developers/README.md

@@ -0,0 +1,13 @@
+# Developers
+
+This part of the book is for understanding Trin, and processes around
+building Trin better.
+
+Where the Trin crates and the Portal Network specification are the source of
+truth, this section seeks to offer a quicker "key concepts" for getting started.
+
+It seeks to answer questions like:
+- What do I need to know about the Portal Network?
+- How do the different components of Trin work together?
+- What sort of data guarantees are made and how are they achieved?
+- What things should a new contributor be mindful of?

book/src/developers/architecture/README.md

@@ -0,0 +1,8 @@
+# Architecture
+
+Trin can be understood a from different perspectives.
+
+- How is code organised?
+- How does data flow through trin?
+- How is data stored?
+- How does testing work?

book/src/developers/architecture/database.md

@@ -0,0 +1,33 @@
+# Database
+
+The database related code is located in `./trin-core/src/portalnet/storage.rs`.
+
+There are three main database kinds:
+
+|DB Name|Kind|Location|Purpose|Keys|Values|
+|-|-|-|-|-|-|
+|Main|RocksDB|Disk|Data store|Content ID|Content data bytes|
+|Memory|HashMap|Memory|Kademlia cache|Content key|Content data bytes|
+|Meta|SQLite|Disk|Manage DB size|Content ID|Content key, content size|
+
+## Main content database
+
+This is a persistent file-based database that uses RocksDB.
+It is also called the "radius" database because content management rules are based on
+the radius of content (specifically the content distance to the node ID).
+
+## Memory content database
+
+This uses is an in-memory hashmap to keep content that may not be required for long term
+storage. An overlay service uses this database when receiving data from a peer as
+part of Kademlia-related actions. If required, data is later moved to disk in the
+main content database.
+
+## Meta database
+
+This is an SQLite database that stores metadata. For a piece of content, this includes
+the content ID, content key and the size of the content. It makes assess the size of
+the main database quicker by avoiding the need to repeatedly compute the size of each content.
+
+Database updates occur in tandum with the main database, where if an operation in one database
+fails, the other can revert the operation to remain synced.

book/src/developers/architecture/process_flow.md

@@ -0,0 +1,109 @@
+# Process flow
+
+The following main threads are spawned when Trin is started via `./src/main.rs`.
+
+```mermaid
+stateDiagram-v2
+    trin: trin
+
+    state trin {
+        utplistner: UTP listner
+        subprotocolhandler: sub-protocol handler
+        subprotocolnetworktask: sub-protocol network task
+        portaleventshandler: portal events handler
+        jsonrpcserver: JSON-RPC server
+
+        main() --> utplistner
+        main() --> subprotocolhandler
+        main() --> subprotocolnetworktask
+        main() --> portaleventshandler
+        main() --> jsonrpcserver
+
+    }
+
+```
+Where for each sub-protocol implemented (History, State, Etc.,), a new thread is started.
+
+Here are some of the major components of trin-core that are called on startup within `./trin-core/src/lib.rs`.
+
+```mermaid
+stateDiagram-v2
+    trincore: trin-core
+    collection: configs and services
+
+    state trin {
+        main() --> from_cli()
+        from_cli() --> run_trin()
+        run_trin() --> discovery()
+        run_trin() --> utp_listener()
+        run_trin() --> header_oracle()
+        run_trin() --> portalnet_config
+        run_trin() --> storage_config
+
+    }
+
+    state trincore {
+        portalnet_config --> collection
+        storage_config --> collection
+        discovery() --> collection
+        header_oracle() --> collection
+        utp_listener() --> collection
+
+
+        state portalnet {
+            portalnet_config
+            storage_config
+            discovery()
+        }
+        state utp {
+            utp_listener()
+        }
+        state validation {
+            header_oracle()
+        }
+    }
+```
+
+Once the initial collection of important configs and services have
+been aggregated, they are passed to the crates for each sub-protocol (`trin-history` shown here). The received data structures are then
+used to start the JSON-RPC server.
+
+An events listener awaits network activity that can be actioned.
+```mermaid
+stateDiagram-v2
+    trincore: trin-core
+    trinhistory: trin-history
+    jsonrpchistory: JSON-RPC History details
+    historyhandler: History handler
+    collection: configs and services
+
+    state trin {
+        collection --> initialize_history_network()
+        collection --> HistoryRequestHandler
+        initialize_history_network() --> jsonrpchistory
+        jsonrpchistory --> launch_jsonrpc_server()
+        HistoryRequestHandler --> historyhandler
+        collection --> events()
+        historyhandler --> events()
+    }
+
+    state trincore {
+        state portalnet {
+            events()
+        }
+
+    }
+    state trinhistory {
+        initialize_history_network()
+        state jsonrpc {
+            HistoryRequestHandler
+        }
+    }
+    state rpc {
+        launch_jsonrpc_server()
+    }
+```
+
+Then `./trin-core/portalnet/events.rs` is handles events at the level of the Portal Wire Protocol.
+These are defined messages that are compliant with the Discv5 protocol, and specific
+to the Portal Network.