diff --git a/adminSiteClient/AdminApp.tsx b/adminSiteClient/AdminApp.tsx index 0f8e46fae0e..32d5ec94223 100644 --- a/adminSiteClient/AdminApp.tsx +++ b/adminSiteClient/AdminApp.tsx @@ -16,6 +16,7 @@ import { RedirectsIndexPage } from "./RedirectsIndexPage.js" import SiteRedirectsIndexPage from "./SiteRedirectsIndexPage" import { TagEditPage } from "./TagEditPage.js" import { TagsIndexPage } from "./TagsIndexPage.js" +import { TagGraphPage } from "./TagGraphPage.js" import { PostsIndexPage } from "./PostsIndexPage.js" import { TestIndexPage } from "./TestIndexPage.js" import { NotFoundPage } from "./NotFoundPage.js" @@ -263,6 +264,11 @@ export class AdminApp extends React.Component<{ /> )} /> + ( Tags +
  • + + Tag Graph + +
  • Bulk downloads diff --git a/adminSiteClient/TagEditPage.tsx b/adminSiteClient/TagEditPage.tsx index 589fff4b8a4..374bab4a4d2 100644 --- a/adminSiteClient/TagEditPage.tsx +++ b/adminSiteClient/TagEditPage.tsx @@ -1,10 +1,10 @@ import React from "react" import { observer } from "mobx-react" -import { observable, computed, action, runInAction } from "mobx" +import { observable, computed, runInAction } from "mobx" import { Prompt, Redirect } from "react-router-dom" import { DbChartTagJoin } from "@ourworldindata/utils" import { AdminLayout } from "./AdminLayout.js" -import { BindString, NumericSelectField, FieldsRow, Timeago } from "./Forms.js" +import { BindString, Timeago } from "./Forms.js" import { DatasetList, DatasetListItem } from "./DatasetList.js" import { ChartList, ChartListItem } from "./ChartList.js" import { TagBadge } from "./TagBadge.js" @@ -12,20 +12,17 @@ import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" interface TagPageData { id: number - parentId?: number name: string specialType?: string updatedAt: string datasets: DatasetListItem[] charts: ChartListItem[] children: DbChartTagJoin[] - possibleParents: DbChartTagJoin[] slug: string | null } class TagEditable { @observable name: string = "" - @observable parentId?: number @observable slug: string | null = null constructor(json: TagPageData) { @@ -84,7 +81,7 @@ class TagEditor extends React.Component<{ tag: TagPageData }> { if ( !window.confirm( - `Really delete the category ${tag.name}? This action cannot be undone!` + `Really delete the tag ${tag.name}? This action cannot be undone!` ) ) return @@ -100,21 +97,6 @@ class TagEditor extends React.Component<{ tag: TagPageData }> { } } - @action.bound onChooseParent(parentId: number) { - if (parentId === -1) { - this.newtag.parentId = undefined - } else { - this.newtag.parentId = parentId - } - } - - @computed get parentTag() { - const { parentId } = this.props.tag - return parentId - ? this.props.tag.possibleParents.find((c) => c.id === parentId) - : undefined - } - render() { const { tag } = this.props const { newtag } = this @@ -142,49 +124,30 @@ class TagEditor extends React.Component<{ tag: TagPageData }> { field="name" store={newtag} label="Name" - helpText="Category names should ideally be unique across the database and able to be understood without context" + helpText="Tag names must be unique and should be able to be understood without context" /> - - ({ - value: p.id as number, - label: p.name, - })) - )} - onValue={this.onChooseParent} - /> -
    -
    - {this.parentTag && ( - - )} -
    -
    {" "} {tag.datasets.length === 0 && tag.children.length === 0 && !tag.specialType && ( )}
    diff --git a/adminSiteClient/TagGraphPage.scss b/adminSiteClient/TagGraphPage.scss new file mode 100644 index 00000000000..66dbae5c7d7 --- /dev/null +++ b/adminSiteClient/TagGraphPage.scss @@ -0,0 +1,113 @@ +.TagGraphPage { + .page-header { + display: flex; + justify-content: space-between; + } + .tag-box { + position: relative; + padding: 8px; + padding-right: 0; + background-color: #f0f0f0; + transition: box-shadow 0.2s; + border-radius: 3px; + border: 3px solid transparent; + cursor: default; + &.tag-box--dragging { + z-index: 2; + box-shadow: 4px 4px 8px 8px #bbb; + } + + &.tag-box--hovering:not(.tag-box--dragging) { + border: 3px solid orange; + } + + // 🤫 + & > .tag-box { + background-color: #d0d0d0; + & > .tag-box { + background-color: #f0f0f0; + & > .tag-box { + background-color: #d0d0d0; + & > .tag-box { + background-color: #f0f0f0; + & > .tag-box { + background-color: #d0d0d0; + & > .tag-box { + background-color: #f0f0f0; + } + } + } + } + } + } + + margin: 4px; + .grip-button { + position: absolute; + right: 4px; + top: 8px; + width: 32px; + height: 32px; + background-color: transparent; + padding: 0; + border: none; + line-height: 1; + transition: 0.2s; + color: #666; + &:hover { + color: #111; + } + } + } + .root-tag-box { + padding: 24px 0; + cursor: default; + > .grip-button { + display: none; + } + } + .tag-box__controls-container { + margin-bottom: 8px; + } + .tag-box__weight-control { + margin: 0 8px; + label { + margin: 0; + margin-right: 8px; + } + input { + width: 50px; + padding-left: 5px; + padding-right: 4px; + border: none; + background-color: transparent; + padding-top: 4px; + padding-bottom: 5px; + margin-right: 0; + } + } + .add-tag-button { + margin-right: 8px; + background-color: #686868; + border-color: #686868; + &:hover { + background-color: #888; + border-color: #888; + } + } + .add-tag-form { + display: inline-block; + margin-right: 8px; + } + .add-tag-input { + width: 256px; + border-radius: 3px; + } + + .TagBadge { + font-size: 1.1em; + font-weight: 500; + background: none; + border: none; + } +} diff --git a/adminSiteClient/TagGraphPage.tsx b/adminSiteClient/TagGraphPage.tsx new file mode 100644 index 00000000000..a1cef3caf5f --- /dev/null +++ b/adminSiteClient/TagGraphPage.tsx @@ -0,0 +1,559 @@ +import React from "react" +import { observer } from "mobx-react" +import { observable, action, runInAction, toJS, computed } from "mobx" +import * as lodash from "lodash" +import { AdminLayout } from "./AdminLayout.js" +import { + MinimalTagWithIsTopic, + TagGraphNode, + TagGraphRoot, + createTagGraph, +} from "@ourworldindata/utils" +import { TagBadge } from "./TagBadge.js" +import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" +import { + DndContext, + DragEndEvent, + pointerWithin, + useDraggable, + useDroppable, +} from "@dnd-kit/core" +import cx from "classnames" +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome" +import { faGrip } from "@fortawesome/free-solid-svg-icons" +import { AutoComplete, Button, Popconfirm } from "antd" +import { FlatTagGraph, FlatTagGraphNode } from "@ourworldindata/types" +import { Link } from "react-router-dom" + +function DraggableDroppable(props: { + id: string + children: React.ReactNode + className?: string + disableDroppable?: boolean +}) { + const drag = useDraggable({ + id: props.id, + }) + const { attributes, listeners, transform } = drag + const shouldDisableDroppable = !!transform || props.disableDroppable + const dragSetNodeRef = drag.setNodeRef + const drop = useDroppable({ + id: props.id, + disabled: shouldDisableDroppable, + }) + const isOver = drop.isOver + const dropSetNodeRef = drop.setNodeRef + + return ( +
    { + dragSetNodeRef(element) + dropSetNodeRef(element) + }} + > + {React.Children.map(props.children, (child) => { + if (React.isValidElement(child)) { + if (child.type === TagGraphNodeContainer) { + return React.cloneElement(child, { + ...child.props, + // If this node is being dragged, prevent all its children from being valid drop targets + disableDroppable: shouldDisableDroppable, + }) + } + } + return child + })} + +
    + ) +} + +@observer +class AddChildForm extends React.Component<{ + tags: MinimalTagWithIsTopic[] + label: string + setChild: (parentId: number, childId: number) => void + parentId: number +}> { + @observable isAddingTag: boolean = false + @observable autocompleteValue: string = "" + + render() { + if (!this.isAddingTag) { + return ( + + ) + } + return ( +
    + (this.autocompleteValue = value)} + options={this.props.tags.map((tag) => ({ + value: tag.name, + label: tag.name, + }))} + filterOption={(inputValue, option) => { + if (!option?.label) return false + return option.label + .toLowerCase() + .startsWith(inputValue.toLowerCase()) + }} + /> + + + + ) + } +} + +@observer +class TagGraphNodeContainer extends React.Component<{ + node: TagGraphNode + parentId: number + setWeight: (parentId: number, childId: number, weight: number) => void + setChild: (parentId: number, childId: number) => void + removeNode: (parentId: number, childId: number) => void + tags: MinimalTagWithIsTopic[] + parentsById: Record + disableDroppable?: boolean +}> { + constructor(props: any) { + super(props) + this.handleUpdateWeight = this.handleUpdateWeight.bind(this) + } + + handleUpdateWeight(e: React.ChangeEvent) { + const weight = Number(e.target.value) + const parentId = this.props.parentId + const childId = this.props.node.id + this.props.setWeight(parentId, childId, weight) + } + + // Coparents are the other parents of this node + // e.g. + // Health -> Obesity + // Food and Agriculture -> Diet -> Obesity + // If we're on the Health -> Obesity node, coparents = [Diet] + @computed get coparents() { + return (this.props.parentsById[this.props.node.id] || []).filter( + (tag) => tag.id !== this.props.parentId + ) + } + + get addableTags() { + return this.props.tags.filter((tag) => { + const isDuplicate = tag.id === this.props.node.id + const isParent = this.props.node.path.includes(tag.id) + const isChild = this.props.node.children.find( + (c) => c.name === tag.name + ) + const isCoparent = this.coparents.find((t) => t.id === tag.id) + return !isDuplicate && !isParent && !isChild && !isCoparent + }) + } + + render() { + const { id, name, path, children, weight, isTopic } = this.props.node + const serializedPath = path.join("-") + return ( + // IDs can't start with a number, so we prefix with "node-" + // Not using data- attributes because they don't work with DndContext + + + {isTopic ? ( + + ⭐️ + + ) : null} + {this.coparents.length > 0 ? ( +

    + This tag is also a child of:{" "} + {this.coparents.map((tag) => tag.name).join(", ")} +

    + ) : null} +
    + + + + + + + + Are you sure you want to remove this tag + relationship? + +
    + Child tag relationships will also be removed + unless they are referenced elsewhere. +
    +
    + } + onConfirm={() => + this.props.removeNode(path.at(-2)!, id) + } + okText="Yes" + cancelText="No" + > + + + + {children.map((node) => ( + + ))} +
    + ) + } +} + +function sortByWeightThenName(a: FlatTagGraphNode, b: FlatTagGraphNode) { + return b.weight - a.weight || a.name.localeCompare(b.name) +} + +// "node-1-2-3" -> [1, 2, 3] +function getPath(elementId?: string): number[] { + if (!elementId) return [] + return elementId.split("-").slice(1).map(Number) +} + +function insertChildAndSort( + children: FlatTagGraphNode[] = [], + newNode: FlatTagGraphNode +): FlatTagGraphNode[] { + return [...children, newNode].sort(sortByWeightThenName) +} + +@observer +export class TagGraphPage extends React.Component { + static contextType = AdminAppContext + context!: AdminAppContextType + + constructor(props: Readonly) { + super(props) + this.handleDragEnd = this.handleDragEnd.bind(this) + this.setWeight = this.setWeight.bind(this) + this.setChild = this.setChild.bind(this) + this.removeNode = this.removeNode.bind(this) + } + + @observable flatTagGraph: FlatTagGraph = {} + @observable rootId: number | null = null + @observable addTagParentId?: number + @observable tags: MinimalTagWithIsTopic[] = [] + + @computed get tagGraph(): TagGraphRoot | null { + if (!this.rootId) return null + return createTagGraph(this.flatTagGraph, this.rootId) + } + + @computed get parentsById(): Record { + if (!this.tagGraph) return {} + const parentsById: Record = {} + for (const [parentId, children] of Object.entries(this.flatTagGraph)) { + for (const child of children) { + const parent = this.tags.find( + (tag) => tag.id === Number(parentId) + ) + if (parent) { + if (!parentsById[child.childId]) { + parentsById[child.childId] = [parent] + } else { + parentsById[child.childId].push(parent) + } + } + } + } + return parentsById + } + + @computed get nonAreaTags() { + if (!this.rootId) return [] + const areaTags = this.flatTagGraph[this.rootId] + const areaTagIds = areaTags?.map((tag) => tag.childId) || [] + return this.tags.filter((tag) => !areaTagIds.includes(tag.id)) + } + + getAllChildrenOfNode(childId: number): FlatTagGraphNode[] { + const allChildren: FlatTagGraphNode[] = + toJS(this.flatTagGraph[childId]) || [] + + for (const child of allChildren) { + if (this.flatTagGraph[child.childId]) { + allChildren.push(...this.flatTagGraph[child.childId]) + } + } + + return allChildren + } + + @action.bound setWeight(parentId: number, childId: number, weight: number) { + const parent = this.flatTagGraph[parentId] + if (!parent) return + const child = parent.find((node) => node.childId === childId) + if (!child) return + child.weight = weight + this.flatTagGraph[parentId] = parent.sort(sortByWeightThenName) + } + + @action.bound setChild(parentId: number, childId: number) { + const siblings = this.flatTagGraph[parentId] + const tag = this.tags.find((tag) => tag.id === childId) + if (!tag) return + const child: FlatTagGraphNode = { + childId: tag.id, + parentId: parentId, + name: tag.name, + weight: 100, + slug: tag.slug, + isTopic: tag.isTopic, + } + if (siblings) { + this.flatTagGraph[parentId] = insertChildAndSort(siblings, child) + } else { + this.flatTagGraph[parentId] = [child] + } + } + + @action.bound removeNode(parentId: number, childId: number) { + const children = this.flatTagGraph[parentId] + if (!children) return + + // Remove the child from its parent. If there are no other children, delete the record + const remainingChildren = this.flatTagGraph[parentId].filter( + (node) => node.childId !== childId + ) + if (remainingChildren.length) { + this.flatTagGraph[parentId] = remainingChildren + } else { + delete this.flatTagGraph[parentId] + } + + // If the child had no other parents, delete its grandchildren also + // i.e. if you delete the "Health" area, the subgraph (Diseases -> Cardiovascular Disease, Cancer) will also be deleted, + // but the subgraph (Air Pollution -> Indoor Air Pollution, Outdoor Air Pollution) won't be, because it's still a child of "Energy and Environment" + const hasMultipleParents = this.parentsById[childId] + if (!hasMultipleParents) return + + const grandchildren = this.flatTagGraph[childId] + if (!grandchildren) return + + for (const grandchild of grandchildren) { + this.removeNode(childId, grandchild.childId) + } + } + + @action.bound handleDragEnd(event: DragEndEvent) { + if (!this.tagGraph) return + + const activeHtmlId = event.active.id as string + const overHtmlId = event.over?.id as string + if (!activeHtmlId || !overHtmlId) return + + const childPath = getPath(activeHtmlId) + const newParentPath = getPath(overHtmlId) + const previousParentPath = childPath.slice(0, -1) + if (!childPath || !newParentPath || !previousParentPath) return + + const childId = childPath.at(-1) + const newParentId = newParentPath.at(-1) + const previousParentId = previousParentPath.at(-1) + if (!childId || !previousParentId || !newParentId) return + + const childNode = this.flatTagGraph[previousParentId].find( + (node) => node.childId === childId + ) + if (!childNode) return + + const isNoop = lodash.isEqual(previousParentPath, newParentPath) + if (isNoop) return + + // Prevents these two structures: + // Parantheses indicate the subgraph that was dagged + // Energy -> (Energy) + // Nuclear Energy -> (Energy -> Nuclear Energy) + const allChildrenOfChild = this.getAllChildrenOfNode(childId) + const isCyclical = [childNode, ...allChildrenOfChild].find((child) => + newParentPath.includes(child.childId) + ) + if (isCyclical) { + alert("This operation would create a cycle") + return + } + const isSibling = this.flatTagGraph[newParentId]?.find( + (node) => node.childId === childId + ) + if (isSibling) { + alert("This parent already has this child") + return + } + + // Add child to new parent + childNode.parentId = newParentId + this.flatTagGraph[newParentId] = insertChildAndSort( + this.flatTagGraph[newParentId], + childNode + ) + + // Remove child from previous parent + this.flatTagGraph[previousParentId] = this.flatTagGraph[ + previousParentId + ].filter((node) => node.childId !== childId) + } + + @action.bound async saveTagGraph() { + if (!this.tagGraph) return + await this.context.admin.requestJSON( + "/api/tagGraph", + { tagGraph: toJS(this.flatTagGraph) }, + "POST" + ) + } + + render() { + return ( + +
    +
    +

    Tag Graph

    +
    + + +
    +
    +

    + Drag and drop tags according to their hierarchy. Top + level tags should be areas. Tags are ordered by weight + (higher weights first) and then by name. To add a new + tag, visit the tags page. +

    + + + {this.tagGraph?.children.map((node) => ( + + ))} + + +
    +
    + ) + } + + async getData() { + const flatTagGraph = await this.context.admin.getJSON< + FlatTagGraph & { + __rootId: number + } + >("/api/flatTagGraph.json") + + const tags = await this.context.admin + .getJSON<{ tags: MinimalTagWithIsTopic[] }>("/api/tags.json") + .then((data) => data.tags) + + runInAction(() => { + const { __rootId, ...rest } = flatTagGraph + this.rootId = __rootId + this.flatTagGraph = rest + this.tags = tags + }) + } + + componentDidMount() { + void this.getData() + } +} diff --git a/adminSiteClient/TagsIndexPage.tsx b/adminSiteClient/TagsIndexPage.tsx index eb9f973717b..c1c503bf03f 100644 --- a/adminSiteClient/TagsIndexPage.tsx +++ b/adminSiteClient/TagsIndexPage.tsx @@ -1,204 +1,109 @@ -import React from "react" import { observer } from "mobx-react" -import { observable, computed, action, runInAction } from "mobx" -import * as lodash from "lodash" -import { Redirect } from "react-router-dom" +import React from "react" import { AdminLayout } from "./AdminLayout.js" -import { FieldsRow, Modal, TextField } from "./Forms.js" -import { DbChartTagJoin } from "@ourworldindata/utils" -import { TagBadge } from "./TagBadge.js" import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" - -interface TagListItem { - id: number - name: string - parentId: number - specialType?: string -} +import { observable, runInAction } from "mobx" +import { DbPlainTag } from "@ourworldindata/types" +import { TagBadge } from "./TagBadge.js" +import { Link } from "react-router-dom" +import { Button, Modal } from "antd" @observer -class AddTagModal extends React.Component<{ - parentId?: number - onClose: () => void -}> { +export class TagsIndexPage extends React.Component { static contextType = AdminAppContext context!: AdminAppContextType - @observable tagName: string = "" - @observable newTagId?: number - - @computed get tag() { - if (!this.tagName) return undefined + @observable tags: DbPlainTag[] = [] + @observable newTagName = "" + @observable newTagSlug = "" + @observable isAddingTag = false - return { - parentId: this.props.parentId, - name: this.tagName, - } + componentDidMount(): void { + void this.getData() + this.addTag = this.addTag.bind(this) } - async submit() { - if (this.tag) { - const resp = await this.context.admin.requestJSON( - "/api/tags/new", - { tag: this.tag }, - "POST" - ) - if (resp.success) { - this.newTagId = resp.tagId - } - } + async getData() { + const result = await this.context.admin.getJSON<{ tags: DbPlainTag[] }>( + "/api/tags.json" + ) + runInAction(() => { + this.tags = result.tags + }) } - @action.bound onTagName(tagName: string) { - this.tagName = tagName + async addTag() { + await this.context.admin.requestJSON( + "/api/tags/new", + { + name: this.newTagName, + slug: this.newTagSlug, + }, + "POST" + ) + this.isAddingTag = false + this.newTagName = "" + this.newTagSlug = "" + await this.getData() } - render() { + addTagModal() { return ( - -
    { - e.preventDefault() - void this.submit() - }} - > -
    -
    Add category
    -
    -
    - -
    -
    - -
    + (this.isAddingTag = false)} + > +

    Add tag

    + + (this.newTagName = e.target.value)} + /> + (this.newTagSlug = e.target.value)} + /> + - {this.newTagId !== undefined && ( - - )}
    ) } -} - -@observer -export class TagsIndexPage extends React.Component { - static contextType = AdminAppContext - context!: AdminAppContextType - - @observable tags: TagListItem[] = [] - @observable isAddingTag: boolean = false - @observable addTagParentId?: number - - @computed get categoriesById(): Record { - return lodash.keyBy(this.tags, (t) => t.id) - } - - @computed get parentCategories(): { - id: number - name: string - specialType?: string - children: TagListItem[] - }[] { - const parentCategories = this.tags - .filter((c) => !c.parentId) - .map((c) => ({ - id: c.id, - name: c.name, - specialType: c.specialType, - children: this.tags.filter((c2) => c2.parentId === c.id), - })) - - return parentCategories - } - - @action.bound onNewTag(parentId?: number) { - this.addTagParentId = parentId - this.isAddingTag = true - } render() { - const { parentCategories } = this - return (
    - - Showing {this.tags.length} tags - + {this.addTagModal()} +
    +

    Tags

    + +

    - Tags are a way of organizing data. Each chart and - dataset can be assigned any number of tags. A tag may be - listed under another parent tag. + This is every single tag we have in the database. To + organise them hierarchically, see the{" "} + tag graph.

    -
    -
    -

    Top-Level Categories

    - {parentCategories.map((parent) => ( - - ))} - -
    - {parentCategories.map((parent) => ( -
    -

    {parent.name}

    - {parent.specialType === "systemParent" && ( -

    - These are special categories that are - assigned automatically. -

    - )} - {parent.children.map((tag) => ( - - ))} - -
    - ))} -
    + {this.tags.map((tag) => ( + + ))}
    - {this.isAddingTag && ( - (this.isAddingTag = false))} - /> - )}
    ) } - - async getData() { - const json = await this.context.admin.getJSON("/api/tags.json") - runInAction(() => { - this.tags = json.tags - }) - } - - componentDidMount() { - void this.getData() - } } diff --git a/adminSiteClient/admin.scss b/adminSiteClient/admin.scss index b6a9507e77c..b8d643ee3a5 100644 --- a/adminSiteClient/admin.scss +++ b/adminSiteClient/admin.scss @@ -12,6 +12,7 @@ @import "codemirror/lib/codemirror.css"; @import "./GdocsIndexPage.scss"; +@import "./TagGraphPage.scss"; html { font-size: 14px; @@ -855,13 +856,6 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) { 0 0 0 0.2rem #fff; } -.TagsIndexPage { - .badge { - font-size: 0.8em; - margin-right: 0.5em; - } -} - .TagBadge { display: inline-block; border-radius: 5px; @@ -1387,3 +1381,19 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) { margin: 1px; } } + +.TagsIndexPage__header { + display: flex; + justify-content: space-between; + p { + line-height: 32px; + margin: 0; + } +} + +.TagsIndexPage__add-tag-modal { + input { + margin-bottom: 1rem; + display: block; + } +} diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 302f85dcc50..5538b330ef2 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -53,6 +53,7 @@ import { pick, Json, checkIsGdocPostExcludingFragments, + checkIsPlainObjectWithGuard, } from "@ourworldindata/utils" import { DbPlainDatasetTag, @@ -76,6 +77,7 @@ import { PostsGdocsTableName, DbPlainDataset, DbInsertUser, + FlatTagGraph, } from "@ourworldindata/types" import { getVariableDataRoute, @@ -1974,8 +1976,8 @@ putRouteWithRWTransaction( const tag = (req.body as { tag: any }).tag await db.knexRaw( trx, - `UPDATE tags SET name=?, updatedAt=?, parentId=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.parentId, tag.slug, tagId] + `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.slug, tagId] ) if (tag.slug) { // See if there's a published gdoc with a matching slug. @@ -1983,13 +1985,14 @@ putRouteWithRWTransaction( // where the page for the topic is just an article. const gdoc = await db.knexRaw>( trx, - `SELECT slug FROM posts_gdocs pg - WHERE EXISTS ( - SELECT 1 - FROM posts_gdocs_x_tags gt - WHERE pg.id = gt.gdocId AND gt.tagId = ? - ) AND pg.published = TRUE`, - [tagId] + `-- sql + SELECT slug FROM posts_gdocs pg + WHERE EXISTS ( + SELECT 1 + FROM posts_gdocs_x_tags gt + WHERE pg.id = gt.gdocId AND gt.tagId = ? + ) AND pg.published = TRUE AND pg.slug = ?`, + [tagId, tag.slug] ) if (!gdoc.length) { return { @@ -2008,30 +2011,48 @@ postRouteWithRWTransaction( apiRouter, "/tags/new", async (req: Request, res, trx) => { - const tag = (req.body as { tag: any }).tag + const tag = req.body + function validateTag( + tag: unknown + ): tag is { name: string; slug: string | null } { + return ( + checkIsPlainObjectWithGuard(tag) && + typeof tag.name === "string" && + (tag.slug === null || + (typeof tag.slug === "string" && tag.slug !== "")) + ) + } + if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) + + const conflictingTag = await db.knexRawFirst<{ + name: string + slug: string | null + }>( + trx, + `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, + [tag.name, tag.slug] + ) + if (conflictingTag) + throw new JsonError( + conflictingTag.name === tag.name + ? `Tag with name ${tag.name} already exists` + : `Tag with slug ${tag.slug} already exists`, + 400 + ) + const now = new Date() const result = await db.knexRawInsert( trx, - `INSERT INTO tags (parentId, name, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - [tag.parentId, tag.name, now, now] + `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + // parentId will be deprecated soon once we migrate fully to the tag graph + [tag.name, tag.slug, now, now] ) return { success: true, tagId: result.insertId } } ) getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { - const tags = await db.knexRaw( - trx, - `-- sql - SELECT t.id, t.name, t.parentId, t.specialType - FROM tags t LEFT JOIN tags p ON t.parentId=p.id - ORDER BY t.name ASC - ` - ) - - return { - tags, - } + return { tags: await db.getMinimalTagsWithIsTopic(trx) } }) deleteRouteWithRWTransaction( @@ -2654,4 +2675,55 @@ getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { return [...generateAllWorkArchieMl()].join("") }) +getRouteWithROTransaction( + apiRouter, + "/flatTagGraph.json", + async (req, res, trx) => { + const flatTagGraph = await db.getFlatTagGraph(trx) + return flatTagGraph + } +) + +postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { + const tagGraph = req.body?.tagGraph as unknown + if (!tagGraph) { + throw new JsonError("No tagGraph provided", 400) + } + + function validateFlatTagGraph( + tagGraph: Record + ): tagGraph is FlatTagGraph { + if (lodash.isObject(tagGraph)) { + for (const [key, value] of Object.entries(tagGraph)) { + if (!lodash.isString(key) && isNaN(Number(key))) { + return false + } + if (!lodash.isArray(value)) { + return false + } + for (const tag of value) { + if ( + !( + checkIsPlainObjectWithGuard(tag) && + lodash.isNumber(tag.weight) && + lodash.isNumber(tag.parentId) && + lodash.isNumber(tag.childId) + ) + ) { + return false + } + } + } + } + + return true + } + const isValid = validateFlatTagGraph(tagGraph) + if (!isValid) { + throw new JsonError("Invalid tag graph provided", 400) + } + await db.updateTagGraph(trx, tagGraph) + res.send({ success: true }) +}) + export { apiRouter } diff --git a/db/db.ts b/db/db.ts index abca1728962..0c6dde33a76 100644 --- a/db/db.ts +++ b/db/db.ts @@ -19,7 +19,12 @@ import { DbEnrichedPostGdoc, DbRawPostGdoc, parsePostsGdocsRow, + TagGraphRootName, + FlatTagGraph, + FlatTagGraphNode, + MinimalTagWithIsTopic, } from "@ourworldindata/types" +import { groupBy } from "lodash" // Return the first match from a mysql query export const closeTypeOrmAndKnexConnections = async (): Promise => { @@ -448,3 +453,158 @@ export const getNonGrapherExplorerViewCount = ( AND grapherId IS NULL` ).then((res) => res?.count ?? 0) } + +/** + * 1. Fetch all records in tag_graph, isTopic = true when there is a published TP/LTP/Article with the same slug as the tag + * 2. Group tags by their parentId + * 3. Return the flat tag graph along with a __rootId property so that the UI knows which record is the root node + */ +export async function getFlatTagGraph(knex: KnexReadonlyTransaction): Promise< + FlatTagGraph & { + __rootId: number + } +> { + const tagGraphByParentId = await knexRaw( + knex, + `-- sql + SELECT + tg.parentId, + tg.childId, + tg.weight, + t.name, + p.slug IS NOT NULL AS isTopic + FROM + tag_graph tg + LEFT JOIN tags t ON + tg.childId = t.id + LEFT JOIN posts_gdocs p ON + t.slug = p.slug AND p.published = 1 AND p.type IN (:types) + -- order by descending weight, tiebreak by name + ORDER BY tg.weight DESC, t.name ASC`, + { + types: [ + OwidGdocType.TopicPage, + OwidGdocType.LinearTopicPage, + // For sub-topics e.g. Nuclear Energy we use the article format + OwidGdocType.Article, + ], + } + ).then((rows) => groupBy(rows, "parentId")) + + const tagGraphRootIdResult = await knexRawFirst<{ + id: number + }>( + knex, + `-- sql + SELECT id FROM tags WHERE name = "${TagGraphRootName}"` + ) + if (!tagGraphRootIdResult) throw new Error("Tag graph root not found") + + return { ...tagGraphByParentId, __rootId: tagGraphRootIdResult.id } +} + +export async function updateTagGraph( + knex: KnexReadWriteTransaction, + tagGraph: FlatTagGraph +): Promise { + const tagGraphRows: { + parentId: number + childId: number + weight: number + }[] = [] + + for (const children of Object.values(tagGraph)) { + for (const child of children) { + tagGraphRows.push({ + parentId: child.parentId, + childId: child.childId, + weight: child.weight, + }) + } + } + + const existingTagGraphRows = await knexRaw<{ + parentId: number + childId: number + weight: number + }>( + knex, + `-- sql + SELECT parentId, childId, weight FROM tag_graph + ` + ) + // Remove rows that are not in the new tag graph + // Add rows that are in the new tag graph but not in the existing tag graph + const rowsToDelete = existingTagGraphRows.filter( + (row) => + !tagGraphRows.some( + (newRow) => + newRow.parentId === row.parentId && + newRow.childId === row.childId && + newRow.weight === row.weight + ) + ) + const rowsToAdd = tagGraphRows.filter( + (newRow) => + !existingTagGraphRows.some( + (row) => + newRow.parentId === row.parentId && + newRow.childId === row.childId && + newRow.weight === row.weight + ) + ) + + if (rowsToDelete.length > 0) { + await knexRaw( + knex, + `-- sql + DELETE FROM tag_graph + WHERE parentId IN (?) + AND childId IN (?) + AND weight IN (?) + `, + [ + rowsToDelete.map((row) => row.parentId), + rowsToDelete.map((row) => row.childId), + rowsToDelete.map((row) => row.weight), + ] + ) + } + + if (rowsToAdd.length > 0) { + await knexRaw( + knex, + `-- sql + INSERT INTO tag_graph (parentId, childId, weight) + VALUES ? + `, + [rowsToAdd.map((row) => [row.parentId, row.childId, row.weight])] + ) + } +} + +export function getMinimalTagsWithIsTopic( + knex: KnexReadonlyTransaction +): Promise { + return knexRaw( + knex, + `-- sql + SELECT t.id, + t.name, + t.slug, + t.slug IS NOT NULL AND MAX(IF(pg.type IN (:types), TRUE, FALSE)) AS isTopic + FROM tags t + LEFT JOIN posts_gdocs_x_tags gt ON t.id = gt.tagId + LEFT JOIN posts_gdocs pg ON gt.gdocId = pg.id + GROUP BY t.id, t.name + ORDER BY t.name ASC + `, + { + types: [ + OwidGdocType.TopicPage, + OwidGdocType.LinearTopicPage, + OwidGdocType.Article, + ], + } + ) +} diff --git a/db/migration/1716401298509-AddTagGraph.ts b/db/migration/1716401298509-AddTagGraph.ts new file mode 100644 index 00000000000..eec805c7732 --- /dev/null +++ b/db/migration/1716401298509-AddTagGraph.ts @@ -0,0 +1,56 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class AddTagGraph1716401298509 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`-- sql + CREATE TABLE tag_graph ( + parentId INT NOT NULL, + childId INT NOT NULL, + weight INT NOT NULL DEFAULT 100, + updatedAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (parentId, childId), + FOREIGN KEY (parentId) REFERENCES tags (id), + FOREIGN KEY (childId) REFERENCES tags (id), + INDEX (childId), + CONSTRAINT chk_no_self_link CHECK (parentId != childId) + )`) + + // create root tag + await queryRunner.query(`-- sql + INSERT INTO tags (name) VALUES ("tag-graph-root")`) + + // populate tag_graph with the existing relationships from the tags table + await queryRunner.query(`-- sql + INSERT INTO tag_graph (parentId, childId) + SELECT parentId, id + FROM tags + WHERE parentId IS NOT NULL`) + + // insert edges between the 10 top level tags and the root tag + await queryRunner.query(`-- sql + INSERT INTO tag_graph (parentId, childId) + SELECT + (SELECT id FROM tags WHERE name = 'tag-graph-root'), + id + FROM tags + WHERE name IN ( + "Population and Demographic Change", + "Health", + "Food and Agriculture", + "Energy and Environment", + "Poverty and Economic Development", + "Education and Knowledge", + "Innovation and Technological Change", + "Living Conditions, Community and Wellbeing", + "Human Rights and Democracy", + "Violence and War" + )`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`-- sql + DROP TABLE tag_graph`) + await queryRunner.query(`-- sql + DELETE FROM tags WHERE name = "tag-graph-root"`) + } +} diff --git a/package.json b/package.json index f7b03a50f55..7d3d8337cc4 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@bugsnag/js": "^7.20.0", "@bugsnag/plugin-express": "^7.19.0", "@bugsnag/plugin-react": "^7.19.0", + "@dnd-kit/core": "^6.1.0", "@fortawesome/fontawesome-svg-core": "^6.5.2", "@fortawesome/free-brands-svg-icons": "^6.5.2", "@fortawesome/free-solid-svg-icons": "^6.5.2", diff --git a/packages/@ourworldindata/types/src/dbTypes/Tags.ts b/packages/@ourworldindata/types/src/dbTypes/Tags.ts index 16693aab526..f87ed76f90f 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Tags.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Tags.ts @@ -3,13 +3,19 @@ export const TagsTableName = "tags" export interface DbInsertTag { createdAt?: Date id?: number - name: string parentId?: number | null + name: string slug?: string | null specialType?: string | null updatedAt?: Date | null } + export type DbPlainTag = Required // For now, this is all the metadata we need for tags in the frontend export type MinimalTag = Pick + +// Used in the tag graph +export type MinimalTagWithIsTopic = MinimalTag & { + isTopic: boolean +} diff --git a/packages/@ourworldindata/types/src/dbTypes/TagsGraph.ts b/packages/@ourworldindata/types/src/dbTypes/TagsGraph.ts new file mode 100644 index 00000000000..3e9aa929bd0 --- /dev/null +++ b/packages/@ourworldindata/types/src/dbTypes/TagsGraph.ts @@ -0,0 +1,8 @@ +/** the entity in the `tags` table */ +export const TagsGraphTableName = "tag_graph" +export interface DbInsertTagGraph { + parentId: number + childId: number + weight?: number +} +export type DbPlainTag = Required diff --git a/packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts b/packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts index f5ac0d9583e..34cf5a8fd8c 100644 --- a/packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts +++ b/packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts @@ -1,3 +1,5 @@ +import { DbPlainTag } from "../dbTypes/Tags.js" + export interface EntryMeta { slug: string title: string @@ -10,6 +12,46 @@ export interface CategoryWithEntries { subcategories?: CategoryWithEntries[] } +export type DbInsertTagGraphNode = { + parentId: number + childId: number + weight?: number + updatedAt?: string +} + +export type DbPlainTagGraphNode = Required + +export const TagGraphRootName = "tag-graph-root" as const + +export type FlatTagGraphNode = Pick & { + weight: number + isTopic: boolean + parentId: number + childId: number +} + +export type FlatTagGraph = Record + +export interface TagGraphNode { + children: TagGraphNode[] + id: number + isTopic: boolean + name: string + path: number[] + slug: string | null + weight: number +} + +export type TagGraphRoot = TagGraphNode & { + children: TagGraphNode[] + id: number + isTopic: false + name: typeof TagGraphRootName + path: [number] + slug: null + weight: 0 +} + export interface PostReference { id: string title: string diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 11123910861..c96a70588c4 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -123,9 +123,16 @@ export { } from "./domainTypes/Layout.js" export { + TagGraphRootName, + type CategoryWithEntries, type EntryMeta, + type FlatTagGraph, + type FlatTagGraphNode, type PostReference, - type CategoryWithEntries, + type TagGraphNode, + type TagGraphRoot, + type DbInsertTagGraphNode, + type DbPlainTagGraphNode, } from "./domainTypes/ContentGraph.js" export { WP_BlockClass, @@ -613,6 +620,7 @@ export { export { type DbInsertTag, type DbPlainTag, + type MinimalTagWithIsTopic, type MinimalTag, TagsTableName, } from "./dbTypes/Tags.js" diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index e1475feb8f1..52a205bf26b 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -174,6 +174,9 @@ import { UserCountryInformation, Time, TimeBound, + TagGraphRoot, + TagGraphRootName, + TagGraphNode, } from "@ourworldindata/types" import { PointVector } from "./PointVector.js" import React from "react" @@ -1913,3 +1916,40 @@ export function commafyNumber(value: number): string { export function isFiniteWithGuard(value: unknown): value is number { return isFinite(value as any) } + +export function createTagGraph( + tagGraphByParentId: Record, + rootId: number +): TagGraphRoot { + const tagGraph: TagGraphRoot = { + id: rootId, + name: TagGraphRootName, + slug: null, + isTopic: false, + path: [rootId], + weight: 0, + children: [], + } + + function recursivelySetChildren(node: TagGraphNode): TagGraphNode { + const children = tagGraphByParentId[node.id] + if (!children) return node + + for (const child of children) { + const childNode: TagGraphNode = { + id: child.childId, + path: [...node.path, child.childId], + name: child.name, + slug: child.slug, + isTopic: child.isTopic, + weight: child.weight, + children: [], + } + + node.children.push(recursivelySetChildren(childNode)) + } + return node + } + + return recursivelySetChildren(tagGraph) as TagGraphRoot +} diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index a06387d836b..2b26d290b3c 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -126,6 +126,7 @@ export { roundDownToNearestHundred, commafyNumber, isFiniteWithGuard, + createTagGraph, } from "./Util.js" export { diff --git a/yarn.lock b/yarn.lock index bb94cb202a9..822007fd5a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1643,6 +1643,42 @@ __metadata: languageName: node linkType: hard +"@dnd-kit/accessibility@npm:^3.1.0": + version: 3.1.0 + resolution: "@dnd-kit/accessibility@npm:3.1.0" + dependencies: + tslib: "npm:^2.0.0" + peerDependencies: + react: ">=16.8.0" + checksum: 10/750a0537877d5dde3753e9ef59d19628b553567e90fc3e3b14a79bded08f47f4a7161bc0d003d7cd6b3bd9e10aa233628dca07d2aa5a2120cac84555ba1653d8 + languageName: node + linkType: hard + +"@dnd-kit/core@npm:^6.1.0": + version: 6.1.0 + resolution: "@dnd-kit/core@npm:6.1.0" + dependencies: + "@dnd-kit/accessibility": "npm:^3.1.0" + "@dnd-kit/utilities": "npm:^3.2.2" + tslib: "npm:^2.0.0" + peerDependencies: + react: ">=16.8.0" + react-dom: ">=16.8.0" + checksum: 10/cf9e99763fbd9220cb6fdde2950c19fdf6248391234f5ee835601814124445fd8a6e4b3f5bc35543c802d359db8cc47f07d87046577adc41952ae981a03fbda0 + languageName: node + linkType: hard + +"@dnd-kit/utilities@npm:^3.2.2": + version: 3.2.2 + resolution: "@dnd-kit/utilities@npm:3.2.2" + dependencies: + tslib: "npm:^2.0.0" + peerDependencies: + react: ">=16.8.0" + checksum: 10/6cfe46a5fcdaced943982e7ae66b08b89235493e106eb5bc833737c25905e13375c6ecc3aa0c357d136cb21dae3966213dba063f19b7a60b1235a29a7b05ff84 + languageName: node + linkType: hard + "@emotion/babel-plugin@npm:^11.10.0": version: 11.10.2 resolution: "@emotion/babel-plugin@npm:11.10.2" @@ -10947,6 +10983,7 @@ __metadata: "@bugsnag/js": "npm:^7.20.0" "@bugsnag/plugin-express": "npm:^7.19.0" "@bugsnag/plugin-react": "npm:^7.19.0" + "@dnd-kit/core": "npm:^6.1.0" "@fortawesome/fontawesome-svg-core": "npm:^6.5.2" "@fortawesome/free-brands-svg-icons": "npm:^6.5.2" "@fortawesome/free-solid-svg-icons": "npm:^6.5.2" @@ -19435,7 +19472,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.2": +"tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.2.0, tslib@npm:^2.3.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.5.0, tslib@npm:^2.6.2": version: 2.6.2 resolution: "tslib@npm:2.6.2" checksum: 10/bd26c22d36736513980091a1e356378e8b662ded04204453d353a7f34a4c21ed0afc59b5f90719d4ba756e581a162ecbf93118dc9c6be5acf70aa309188166ca