From 01daffd8dfe164a4d5ee246b098b9c05b5933f9e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Apr 2022 12:37:04 -0700 Subject: [PATCH] Save them to packages/ and jq them Refs https://github.com/simonw/datasette.io/issues/98 --- .github/workflows/fetch.yml | 28 +++++++++ fetch.sh | 6 ++ packages.txt | 119 ++++++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 .github/workflows/fetch.yml create mode 100755 fetch.sh create mode 100644 packages.txt diff --git a/.github/workflows/fetch.yml b/.github/workflows/fetch.yml new file mode 100644 index 0000000..18ce92f --- /dev/null +++ b/.github/workflows/fetch.yml @@ -0,0 +1,28 @@ +name: Fetch latest data + +on: + push: + workflow_dispatch: + schedule: + - cron: '51 3 * * *' # Once a day + +jobs: + fetch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Fetch + run: ./fetch.sh + - name: Commit and push + run: |- + git config user.name "Automated" + git config user.email "actions@users.noreply.github.com" + git add -A + timestamp=$(date -u) + git commit -m "${timestamp}" || exit 0 + git pull --rebase + git push diff --git a/fetch.sh b/fetch.sh new file mode 100755 index 0000000..7b7435c --- /dev/null +++ b/fetch.sh @@ -0,0 +1,6 @@ +#!/bin/bash +mkdir -p packages +while read package; do + curl -s "https://pypi.org/pypi/$package/json" | jq > "packages/$package.json" + sleep 1 +done < packages.txt diff --git a/packages.txt b/packages.txt new file mode 100644 index 0000000..3dcf02b --- /dev/null +++ b/packages.txt @@ -0,0 +1,119 @@ +airtable-export +csv-diff +csvs-to-sqlite +datasette +datasette-atom +datasette-auth-existing-cookies +datasette-auth-github +datasette-auth-passwords +datasette-auth-tokens +datasette-auth0 +datasette-backup +datasette-basemap +datasette-block +datasette-block-robots +datasette-bplist +datasette-clone +datasette-cluster-map +datasette-column-inspect +datasette-configure-asgi +datasette-configure-fts +datasette-copyable +datasette-cors +datasette-css-properties +datasette-dateutil +datasette-debug-asgi +datasette-edit-schema +datasette-export-notebook +datasette-geojson +datasette-geojson-map +datasette-glitch +datasette-graphql +datasette-hashed-urls +datasette-haversine +datasette-hovercards +datasette-ics +datasette-import-table +datasette-indieauth +datasette-init +datasette-insert +datasette-jellyfish +datasette-jq +datasette-json-html +datasette-jupyterlite +datasette-leaflet +datasette-leaflet-freedraw +datasette-leaflet-geojson +datasette-mask-columns +datasette-media +datasette-packages +datasette-permissions-sql +datasette-placekey +datasette-pretty-json +datasette-pretty-traces +datasette-psutil +datasette-publish-fly +datasette-publish-vercel +datasette-pyinstrument +datasette-query-history +datasette-query-links +datasette-redirect-forbidden +datasette-redirect-to-https +datasette-remote-metadata +datasette-render-binary +datasette-render-html +datasette-render-images +datasette-render-markdown +datasette-render-timestamps +datasette-ripgrep +datasette-rure +datasette-saved-queries +datasette-schema-versions +datasette-seaborn +datasette-search-all +datasette-sentry +datasette-show-errors +datasette-sqlite-fts4 +datasette-template-request +datasette-template-sql +datasette-tiddlywiki +datasette-tiles +datasette-upload-csvs +datasette-vega +datasette-verify +datasette-write +datasette-x-forwarded-host +datasette-yaml +db-to-sqlite +dbf-to-sqlite +django-sql-dashboard +dogsheep-beta +dogsheep-photos +download-tiles +evernote-to-sqlite +fec-to-sqlite +genome-to-sqlite +geocode-sqlite +geojson-to-sqlite +git-history +github-to-sqlite +google-drive-to-sqlite +google-takeout-to-sqlite +hacker-news-to-sqlite +healthkit-to-sqlite +inaturalist-to-sqlite +markdown-to-sqlite +pocket-to-sqlite +s3-credentials +shapefile-to-sqlite +shot-scraper +sphinx-to-sqlite +sqlite-colorbrewer +sqlite-diffable +sqlite-generate +sqlite-transform +sqlite-utils +swarm-to-sqlite +tableau-to-sqlite +twitter-to-sqlite +yaml-to-sqlite