Get an array of results from GPT (#184)

* cherry-pick some changes from more-intersections branch

* cleanup

* upate test logs

* round more

* ruff check

* rv another change

* data update, pare back logs

* extract array from geogpt

* prompt that ran on 17k images

* filter out vague place names

* plural locatables

* track some stats; I am on an old branch grrr

* try that again

* filter a few bad intersections for Google

* exclude known bad from addresses, too

* allow river cross-streets

* strip most photographer addresses

* extract a few more place names

* array3

* add a few more photographer addresses

* failing tests for direction preservation

* tests pass

* tests pass

* cursed intersections

* update gpt-geocodes with new photographers

* fix distance bug, tweaks to diff_geojson

* geojson change review UI

* template fixes

* drop one more photographer

* test logs

* data update

* update tests

* update geocache

* fix type errors

* update geocode performance stats

* update site

* refactor geocode.py

data/self-hosted-sizes.txt

@@ -1,6 +1,8 @@
 104430,760,495
 104495,760,493
 104508,760,494
+104563,760,497
+104576,491,760
 104578,760,509
 104727,497,760
 104909,760,494
@@ -12,24 +14,29 @@
 104946,760,497
 105012,760,495
 105020,760,496
+105070,760,498
+105074,760,495
+105084,760,491
 105086,760,492
 105087,760,494
 105089,760,483
 105096,760,486
 105109,760,493
 105175,760,497
 1113273,760,256
+1113274,760,248
 1113275,760,253
+1113276,760,252
 1113277,760,252
-1113278,760,254
 1113279,760,253
 1113280,760,258
 1113281,760,254
 1113282,760,257
 1113283,760,261
 1113285,760,257
-1113287,760,259
+1113286,760,262
 1113288,760,264
+1113289,760,261
 1113290,760,260
 1113291,760,259
 1113292,760,256
@@ -48,6 +55,7 @@
 1635948,760,586
 1635949,618,760
 1635950,760,596
+1636100,760,510
 1648987,609,760
 1648988,760,606
 1648989,760,604
@@ -57,7 +65,7 @@
 1648993,611,760
 1648994,605,760
 1648995,604,760
-1648996,604,760
+1663998,760,533
 1691046,591,760
 1691049,587,760
 1691063,760,589
@@ -71,6 +79,7 @@
 1823530,760,617
 1823531,610,760
 1823534,760,618
+1823536,760,624
 1823547,760,619
 1823555,760,619
 1823556,760,619
@@ -93,6 +102,7 @@
 2040737,760,596
 2040738,760,605
 2040741,614,760
+2040746,760,595
 2040755,760,561
 2040759,760,567
 2040760,760,553
@@ -132,14 +142,15 @@
 2040871,760,617
 2040872,760,617
 2040873,760,618
-2040885,760,675
 3984206,760,545
+3984207,760,544
 3984209,760,546
 3984214,544,760
 3984216,546,760
 3984217,760,550
 3984220,760,546
 3984221,548,760
+3984226,760,546
 3984227,546,760
 3984228,546,760
 3984229,760,548
@@ -169,6 +180,10 @@
 3984266,760,546
 3984267,760,546
 3984268,760,549
+3984271,546,760
+3984272,548,760
+3984273,760,546
+3984277,760,543
 3984291,546,760
 3984295,546,760
 3984300,546,760
@@ -177,9 +192,11 @@
 3984305,760,544
 3984314,760,546
 3984323,760,546
+3984329,760,549
 3984330,760,544
 3984331,760,547
 3984332,546,760
+3984333,760,546
 3984336,546,760
 3984351,760,546
 3984352,760,546
@@ -189,6 +206,7 @@
 3984366,760,546
 3984367,760,546
 3984369,546,760
+3984370,546,760
 3984377,760,546
 3984379,546,760
 3984383,760,546
@@ -207,6 +225,7 @@
 3984418,760,546
 3984419,760,546
 3984424,546,760
+3984425,546,760
 3984426,546,760
 3984436,760,546
 3984438,546,760
@@ -233,6 +252,7 @@
 3984473,760,546
 3984478,760,546
 3984479,548,760
+3984486,549,760
 3984491,549,760
 3984496,549,760
 3984497,549,760
@@ -277,6 +297,7 @@
 3984567,549,760
 3984573,542,760
 3984575,549,760
+3984583,549,760
 3984585,546,760
 3984612,760,549
 3984615,549,760
@@ -321,6 +342,9 @@
 3984721,547,760
 3984725,545,760
 3984732,545,760
+3984734,545,760
+3984735,545,760
+3984736,548,760
 3984755,760,544
 3984759,550,760
 3984760,760,546
@@ -338,13 +362,17 @@
 3984786,549,760
 3984789,545,760
 3984791,545,760
+3984793,545,760
 3984794,549,760
 3984807,541,760
+3984812,546,760
 3984816,548,760
 3984818,545,760
 3984819,546,760
 3984841,545,760
 3984845,545,760
+3984851,544,760
+3984852,546,760
 3984853,542,760
 3984855,544,760
 3984857,540,760
@@ -356,21 +384,24 @@
 3984868,545,760
 3984874,544,760
 3984890,546,760
-3984893,545,760
 3984894,548,760
+3984900,550,760
 3984901,546,760
 3984905,760,546
 3984906,550,760
 3984909,547,760
 3984910,550,760
+3984912,547,760
 3984925,550,760
 3984930,550,760
 3984933,550,760
 3984934,550,760
 3984936,543,760
 3984950,548,760
 3984962,548,760
+3984974,550,760
 3984976,760,550
+3984980,550,760
 3984981,545,760
 3984982,548,760
 3984992,550,760
@@ -441,6 +472,7 @@
 3985153,552,760
 3985157,548,760
 3985162,544,760
+3985163,550,760
 3985164,545,760
 3985166,548,760
 3985173,760,551
@@ -468,6 +500,7 @@
 3985206,549,760
 3985208,543,760
 3985209,760,549
+3985219,548,760
 3985221,548,760
 3985222,545,760
 3985223,548,760
@@ -485,6 +518,8 @@
 3985255,545,760
 3985260,760,551
 3985262,550,760
+3985267,546,760
+3985268,760,550
 3985270,548,760
 3985271,545,760
 3985277,760,545
@@ -506,7 +541,9 @@
 416436,760,671
 416437,760,649
 416438,760,659
+416439,760,657
 416440,760,667
+416441,759,659
 416442,760,638
 416443,760,647
 416444,760,638
@@ -516,9 +553,11 @@
 416448,760,632
 416449,760,649
 416450,760,639
+416451,760,641
 416453,760,662
 416454,760,646
 416455,760,641
+416456,760,647
 416457,760,640
 416458,760,652
 416459,760,642
@@ -730,6 +769,7 @@
 417426,760,651
 417427,760,694
 417428,760,661
+417429,760,691
 417430,760,655
 417431,760,672
 417432,760,662
@@ -787,7 +827,6 @@
 485785,760,512
 485786,760,513
 485787,504,760
-485789,511,760
 485793,760,510
 485794,760,509
 485795,760,506
@@ -815,11 +854,13 @@
 485820,508,760
 485821,760,514
 485822,506,760
+485823,760,504
 485824,760,508
 485825,760,507
 485826,760,504
 485827,760,507
 485828,508,760
+485829,504,760
 485830,760,507
 485831,760,511
 485832,760,509
@@ -847,19 +888,20 @@
 485859,509,760
 485860,760,513
 485862,760,510
+485863,760,504
 485864,760,512
 485865,760,515
 485866,760,514
 485867,760,509
 485868,760,513
-485869,503,760
 485870,760,512
 485871,509,760
 485872,760,503
 485873,504,760
 485874,760,512
 485875,502,760
 485876,760,512
+485877,760,498
 485880,760,510
 485881,760,509
 485882,510,760
@@ -934,6 +976,7 @@
 485973,760,509
 485974,760,517
 485976,517,760
+485977,760,509
 485978,760,511
 485979,760,512
 485980,760,510

oldnyc/geocode/calculate_metrics.py

@@ -36,7 +36,7 @@ def diff_geocode(truth_coord, computed_coord):
         return (False, "Missing geocode", "")
     if truth_coord is None:
         return (False, "Should be missing geocode", "")
-    distance_km = haversine(truth_coord, computed_coord)
+    distance_km = haversine(truth_coord[::-1], computed_coord[::-1])
     if distance_km > 0.25:
         return (False, "Too far: %.3f km" % distance_km, f"{distance_km:.3f}")
     return (True, "", f"{distance_km:.3f}")

oldnyc/geocode/coders/gpt.py

@@ -10,7 +10,7 @@
 
 
 class GptCoder(Coder):
-    queries: dict[str, GptResponse]
+    queries: dict[str, list[GptResponse]]
 
     def __init__(self):
         with open("data/gpt-geocodes.json") as f:
@@ -27,9 +27,12 @@ def __init__(self):
     def code_record(self, r: Item):
         # GPT location extractions are always based on record ID, not photo ID.
         id = r.id.split("-")[0]
-        q = self.queries.get(id)
-        if not q:
+        qs = self.queries.get(id)
+        if not qs:
             return None
+        return [locatable for q in qs if (locatable := self.code_one(r, q))]
+
+    def code_one(self, r: Item, q: GptResponse):
         # sys.stderr.write(f"GPT location: {r.id} {q}\n")
 
         if q["type"] in ("no location information", "not in NYC"):

oldnyc/geocode/coders/special_cases.py

@@ -10,6 +10,11 @@ def __init__(self):
         self.counts = Counter[str]()
 
     def code_record(self, r: Item):
+        loc = self.code_one_record(r)
+        if loc:
+            return [loc]
+
+    def code_one_record(self, r: Item):
         if r.title.startswith("Newspapers - China Daily News"):
             # The 2013 Milstein CSV file has a bunch of addresses along Mott Street for these.
             self.counts["China Daily News"] += 1

oldnyc/geocode/coders/subjects.py

@@ -19,6 +19,7 @@
 
 from oldnyc.geocode.geocode_types import Coder, LatLngLocation
 from oldnyc.geojson_utils import assert_point
+from oldnyc.item import Item
 
 # TODO: use subjects.geojson instead of these lists
 parks = {
@@ -289,7 +290,12 @@ def __init__(self):
 
         self.counters = Counter()
 
-    def code_record(self, r):
+    def code_record(self, r: Item):
+        loc = self.code_one_record(r)
+        if loc:
+            return [loc]
+
+    def code_one_record(self, r: Item):
         matches = [
             (geo, spec_pt)
             for geo in r.subject.geographic