rewrite everything now that the import is done

.gitattributes

@@ -0,0 +1,2 @@
+*.sparql linguist-detectable
+*.overpassql linguist-detectable

.github/workflows/ci.yml

@@ -14,29 +14,36 @@ jobs:
       - name: ⏬ Checkout code
         uses: actions/checkout@v3
 
+      - name: ⏬ Enable Corepack
+        run: corepack enable
+
       - name: 🔢 Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+          cache: yarn
 
       - name: ⏬ Install
         run: |
           yarn
-          sh src/stage1-download/index.sh
 
       - name: ✨ Lint
         run: |
           yarn lint
 
       - name: 🧪 Test
         run: |
-          mkdir -p tmp out
           yarn test
-        env:
-          FORCE_COLOR: 1
 
-      - name: 📈 Coveralls
-        uses: coverallsapp/[email protected]
+      - name: 🛠 Build Client
+        run: |
+          cd client
+          yarn build
+
+      - name: ⚛ Deploy Client
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: JamesIves/[email protected]
         with:
-          github-token: ${{ secrets.github_token }}
-        continue-on-error: true
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          BRANCH: gh-pages
+          FOLDER: client/dist

.github/workflows/keepalive.yml

@@ -0,0 +1,15 @@
+name: keepalive
+
+on:
+  schedule:
+    - cron: '8 8 8 * *' # monthly on the 8th
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  keepalive:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: k-yle/keepalive@v1

.github/workflows/sync.yml

@@ -0,0 +1,38 @@
+name: Weekly Sync
+
+on:
+  schedule:
+    - cron: '6 18 * * 4'
+    # at 06:06am on Thursday NZST
+  workflow_dispatch:
+
+jobs:
+  changelog:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [20.x]
+
+    steps:
+      - name: ⏬ Checkout code
+        uses: actions/checkout@v3
+
+      - name: ⏬ Enable Corepack
+        run: corepack enable
+
+      - name: 🔢 Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: yarn
+
+      - name: ⏬ Install
+        run: |
+          yarn
+
+      - name: 🚚 Run the conflation
+        run: |
+          yarn start
+        env:
+          AZ_CON: ${{ secrets.AZ_CON }}

.gitignore

@@ -1,4 +1,7 @@
 node_modules
 tmp
 out
+dist
 coverage
+.yarn/*
+.env.local

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021-2023 Kyℓe Hensel
+Copyright (c) 2022-2024 Kyℓe Hensel
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,31 +1,53 @@
 # NZGB ➡️ OSM place name conflation
 
 [![Test](https://github.com/osm-nz/place-name-conflation/actions/workflows/ci.yml/badge.svg)](https://github.com/osm-nz/place-name-conflation/actions/workflows/ci.yml)
-[![Coverage Status](https://coveralls.io/repos/github/osm-nz/place-name-conflation/badge.svg?branch=main)](https://coveralls.io/github/osm-nz/place-name-conflation?branch=main)
+![](https://github.com/osm-nz/place-name-conflation/workflows/Weekly%20Sync/badge.svg)
 ![Lines of code](https://img.shields.io/tokei/lines/github/osm-nz/place-name-conflation?color=green)
 
 This repository contains the code that regularly compares place names in [OpenStreetMap](https://openstreetmap.org) with the [New Zealand Geographic Board (Ngā Pou Taunaha o Aotearoa) Gazetteer](https://gazetteer.linz.govt.nz).
 
 How this works:
 
+- Once a week, the following process is automatically run:
 - The list of place names is downloaded from the [NZGB](https://gazetteer.linz.govt.nz/gaz.csv)
 - An extract of the OpenStreetMap planet is downloaded (containing only Oceania & Antarctica)
+- All wikidata features that are linked to the NZGB are downloaded
 - The OpenStreetMap planet is compared with the gazetteer, and any discrepancies are identified and categorized
 - The result of the conflation process is converted into a format that allows OpenStreetMap mappers to review each place using the same tools as the [LINZ-to-OSM address import](https://github.com/osm-nz/linz-address-import).
 
+```mermaid
+flowchart LR
+  OSM -->|wikidata| Wikidata
+  Wikidata -->|P5104| NZGB
+  OSM -->|ref:linz:place_id| NZGB
+```
+
+# Usage
+
+If you want to help by adding missing place names, visit [✨the dashboard✨](https://osm-nz.github.io/place-name-conflation).
+
 # Setup
 
 If you want to contribute to the code, the following needs to be done manually:
 
 1. Clone this repository
-1. Download [nodejs v18](https://nodejs.org) or later
-1. Install `yarn` (run `npm i -g yarn`)
-1. Run `yarn`
-1. Run `yarn 1` to download the data (10mins)
-1. Manually download the planet ([West side](https://app.protomaps.com/downloads/osm/6ce28ea0-70a3-4d9d-baf9-abbc3ea93f28) as `tmp/osm-west.csv` and [East side](https://app.protomaps.com/downloads/osm/3c51acd6-ac4f-448a-b36e-7c09cdb09c40) as `tmp/osm-east.csv`)
-1. Manually download [NZGB Lines](https://data.linz.govt.nz/layer/52423) as `tmp/nzgb-lines.csv`
-1. Manually download [NZGB Areas](https://data.linz.govt.nz/layer/52424) as `tmp/nzgb-areas.csv`
-1. Run `yarn 2` to preprocess the data (3mins)
-1. Run `yarn 3` to conflate the data (2mins)
-1. Run `yarn 4` to generate the HTML report (1sec)
-1. Use the generated [`osmPatch`](https://github.com/osm-nz/linz-address-import/blob/main/SPEC.md) files to update the place names in OSM
+2. Download [nodejs v20.11](https://nodejs.org) or later
+3. Run `corepack enable`
+4. Run `yarn`
+5. Run `yarn start`
+6. Use the generated [`osmPatch`](https://github.com/osm-nz/linz-address-import/blob/main/SPEC.md) files to update the place names in OSM
+7. Or run the website locally by running `cd client && yarn start`
+
+# The original import
+
+This codebase originally included a lot of complicated logic, which was used to speed up the one-off effort of importing 30,000 place names.
+Now that the initial import is complete, this logic has been removed to simplify and speed up the process, so that it can run automatically every week.
+
+Some of the removed logic includes:
+
+- Automatically detecting matching features with no [`ref:linz:place_id`](https://osm.wiki/Key:ref:linz:place_id) tag
+- Logic to guess whether a name is English or te reo Māori ([moved to a separate repository](https://github.com/k-yle/name-mi))
+- A system for extracting [`name:etymology`](https://osm.wiki/Key:name:etymology) from the feature's description
+- A system for importing the geometry of lines/areas that were missing using [the L](https://data.linz.govt.nz/layer/52423)[DS](https://data.linz.govt.nz/layer/52424)
+
+See the [`legacy` tag](https://github.com/osm-nz/place-name-conflation/tags/legacy) for the final version of the codebase which included these features.

client/index.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Place Name Conflation</title>
+    <link rel="icon" href="/img/logo.png" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+  </head>
+  <body>
+    <main></main>
+    <script type="module">
+      import { createElement } from 'react';
+      import { createRoot } from 'react-dom/client';
+      import { App } from './src/App';
+
+      createRoot(document.querySelector('main')).render(createElement(App));
+    </script>
+  </body>
+</html>

client/package.json

@@ -0,0 +1,28 @@
+{
+  "private": true,
+  "scripts": {
+    "start": "vite",
+    "build": "vite build"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@mui/icons-material": "^5.15.19",
+    "@mui/material": "^5.15.19",
+    "@mui/x-data-grid": "^7.6.1",
+    "leaflet": "^1.9.4",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-leaflet": "^4.2.1",
+    "react-leaflet-cluster": "^2.1.0",
+    "react-router-dom": "^6.23.1",
+    "react-timeago-i18n": "^2.1.1"
+  },
+  "devDependencies": {
+    "@types/leaflet": "^1.9.12",
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.0",
+    "vite": "^5.2.12"
+  }
+}

client/src/App.tsx

@@ -0,0 +1,56 @@
+import { StrictMode, useEffect, useState } from 'react';
+import { Alert, CircularProgress, CssBaseline } from '@mui/material';
+import { BrowserRouter, Route, Routes } from 'react-router-dom';
+import type { Output } from './types';
+import { DataContext } from './context';
+import { Navbar } from './components/Navbar';
+import { Home } from './pages/Home';
+import { OtherPatchFiles } from './pages/OtherPatchFiles';
+import { Warnings } from './pages/Warnings';
+import './main.css';
+import { LayerInfo } from './pages/LayerInfo';
+
+const BASE_URL = localStorage.dev
+  ? 'http://localhost:3000'
+  : 'https://linz-addr-cdn.kyle.kiwi';
+
+export const App: React.FC = () => {
+  const [data, setData] = useState<Output>();
+  const [error, setError] = useState<unknown>();
+
+  useEffect(() => {
+    fetch(`${BASE_URL}/place-names.osmPatch.geo.json`)
+      .then((r) => r.json())
+      .then(setData)
+      .catch(setError);
+  }, []);
+
+  if (error) {
+    return (
+      <Alert severity="error" style={{ margin: 32 }}>
+        Failed to load data.
+      </Alert>
+    );
+  }
+
+  if (!data) return <CircularProgress style={{ margin: 32 }} />;
+
+  return (
+    <StrictMode>
+      <BrowserRouter>
+        <DataContext.Provider value={data}>
+          <CssBaseline />
+          <Navbar />
+          <main style={{ height: 'calc(100vh - 69px)', overflowY: 'auto' }}>
+            <Routes>
+              <Route path="/" element={<Home />} />
+              <Route path="/other" element={<OtherPatchFiles />} />
+              <Route path="/warnings" element={<Warnings />} />
+              <Route path="/layers" element={<LayerInfo />} />
+            </Routes>
+          </main>
+        </DataContext.Provider>
+      </BrowserRouter>
+    </StrictMode>
+  );
+};

client/src/components/BigNumber.tsx

@@ -0,0 +1,18 @@
+import { Paper } from '@mui/material';
+
+export const BigNumber: React.FC<{
+  mainStat: React.ReactNode;
+  subStat: React.ReactNode;
+  label: React.ReactNode;
+  colour: string;
+}> = ({ colour, mainStat, subStat, label }) => {
+  return (
+    <Paper elevation={8} className="big-number">
+      <main style={{ color: colour }}>{mainStat}</main>
+      <aside>
+        <span>{label}</span>
+        {subStat}
+      </aside>
+    </Paper>
+  );
+};