Add setting to color by query score per alignment for dotplot, suppor…

…t HTML in config slot descriptions (#2483)

* Calculate the mean identity for each query per ref based on concepts from dotPlotly

* Allow HTML in configuration description

* Avoid doing group by weighted means unless explicitly set to color by this setting

packages/core/pluggableElementTypes/renderers/ComparativeServerSideRendererType.ts

@@ -93,14 +93,13 @@ export default class ComparativeServerSideRenderer extends ServerSideRenderer {
    * @returns true if this feature passes all configured filters
    */
   featurePassesFilters(renderArgs: RenderArgsDeserialized, feature: Feature) {
-    if (!renderArgs.filters) {
-      return true
-    }
-    return renderArgs.filters.passes(feature, renderArgs)
+    return renderArgs.filters
+      ? renderArgs.filters.passes(feature, renderArgs)
+      : true
   }
 
   async getFeatures(renderArgs: any) {
-    const { signal, sessionId, adapterConfig } = renderArgs
+    const { sessionId, adapterConfig } = renderArgs
     const { dataAdapter } = await getAdapter(
       this.pluginManager,
       sessionId,
@@ -131,16 +130,10 @@ export default class ComparativeServerSideRenderer extends ServerSideRenderer {
     })
 
     // note that getFeaturesInMultipleRegions does not do glyph expansion
-    const featureObservable = (
-      dataAdapter as BaseFeatureDataAdapter
-    ).getFeaturesInMultipleRegions(requestRegions, {
-      signal,
-    })
-
-    return featureObservable
+    return (dataAdapter as BaseFeatureDataAdapter)
+      .getFeaturesInMultipleRegions(requestRegions, renderArgs)
       .pipe(
-        // @ts-ignore
-        filter(feature => this.featurePassesFilters(renderArgs, feature)),
+        filter(f => this.featurePassesFilters(renderArgs, f)),
         toArray(),
       )
       .toPromise()

plugins/comparative-adapters/src/PAFAdapter/PAFAdapter.ts

@@ -6,7 +6,11 @@ import { Region } from '@jbrowse/core/util/types'
 import { doesIntersect2 } from '@jbrowse/core/util/range'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
-import SimpleFeature, { Feature } from '@jbrowse/core/util/simpleFeature'
+import { SimpleFeature, Feature } from '@jbrowse/core/util'
+import {
+  AnyConfigurationModel,
+  readConfObject,
+} from '@jbrowse/core/configuration'
 import { unzip } from '@gmod/bgzf-filehandle'
 
 export interface PAFRecord {
@@ -21,13 +25,114 @@ export interface PAFRecord {
     blockLen?: number
     mappingQual: number
     numMatches?: number
+    meanScore?: number
   }
 }
 
 function isGzip(buf: Buffer) {
   return buf[0] === 31 && buf[1] === 139 && buf[2] === 8
 }
 
+function zip(a: number[], b: number[]) {
+  return a.map((e, i) => [e, b[i]] as [number, number])
+}
+
+// based on "weighted mean" method from dotPlotly
+// https://github.com/tpoorten/dotPlotly
+// License for dotPlotly reproduced here
+//
+// MIT License
+
+// Copyright (c) 2017 Tom Poorten
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+function getWeightedMeans(ret: PAFRecord[]) {
+  // in the weighted mean longer alignments factor in more
+  // heavily of all the fragments of a query vs the reference that it mapped
+  // to
+  //
+  // this uses a combined key query+'-'+ref to iteratively map all the
+  // alignments that match a particular ref from a particular query (so 1d
+  // array of what could be a 2d map)
+  //
+  // the result is a single number that says e.g. chr5 from human mapped to
+  // chr5 on mouse with 0.8 quality, and that0.8 is then attached to all the
+  // pieces of chr5 on human that mapped to chr5 on mouse. if chr5 on human
+  // also more weakly mapped to chr6 on mouse, then it would have another
+  // value e.g. 0.6. this can show strong and weak levels of synteny,
+  // especially in polyploidy situations
+  const scoreMap: { [key: string]: { quals: number[]; len: number[] } } = {}
+  for (let i = 0; i < ret.length; i++) {
+    const entry = ret[i]
+    const query = entry.qname
+    const target = entry.tname
+    const key = query + '-' + target
+    if (!scoreMap[key]) {
+      scoreMap[key] = { quals: [], len: [] }
+    }
+    scoreMap[key].quals.push(entry.extra.mappingQual)
+    scoreMap[key].len.push(entry.extra.blockLen || 1)
+  }
+
+  const meanScoreMap = Object.fromEntries(
+    Object.entries(scoreMap).map(([key, val]) => {
+      const vals = zip(val.quals, val.len)
+      return [key, weightedMean(vals)]
+    }),
+  )
+  for (let i = 0; i < ret.length; i++) {
+    const entry = ret[i]
+    const query = entry.qname
+    const target = entry.tname
+    const key = query + '-' + target
+    entry.extra.meanScore = meanScoreMap[key]
+  }
+
+  let min = 10000
+  let max = 0
+  for (let i = 0; i < ret.length; i++) {
+    const entry = ret[i]
+    min = Math.min(entry.extra.meanScore || 0, min)
+    max = Math.max(entry.extra.meanScore || 0, max)
+  }
+  for (let i = 0; i < ret.length; i++) {
+    const entry = ret[i]
+    const b = entry.extra.meanScore || 0
+    entry.extra.meanScore = (b - min) / (max - min)
+  }
+
+  return ret
+}
+
+// https://gist.github.com/stekhn/a12ed417e91f90ecec14bcfa4c2ae16a
+function weightedMean(tuples: [number, number][]) {
+  const [valueSum, weightSum] = tuples.reduce(
+    ([valueSum, weightSum], [value, weight]) => [
+      valueSum + value * weight,
+      weightSum + weight,
+    ],
+    [0, 0],
+  )
+  return valueSum / weightSum
+}
+
 export default class PAFAdapter extends BaseFeatureDataAdapter {
   private setupP?: Promise<PAFRecord[]>
 
@@ -123,28 +228,30 @@ export default class PAFAdapter extends BaseFeatureDataAdapter {
   async getRefNames(opts: BaseOptions = {}) {
     // @ts-ignore
     const r1 = opts.regions?.[0].assemblyName
-    const feats = await this.setup()
+    const feats = await this.setup(opts)
 
     const idx = this.getAssemblyNames().indexOf(r1)
     if (idx !== -1) {
       const set = new Set<string>()
       for (let i = 0; i < feats.length; i++) {
-        const f = feats[i]
-        if (idx === 0) {
-          set.add(f.qname)
-        } else {
-          set.add(f.tname)
-        }
+        set.add(idx === 0 ? feats[i].qname : feats[i].tname)
       }
       return Array.from(set)
     }
     console.warn('Unable to do ref renaming on adapter')
     return []
   }
 
-  getFeatures(region: Region, opts: BaseOptions = {}) {
+  getFeatures(
+    region: Region,
+    opts: BaseOptions & { config?: AnyConfigurationModel } = {},
+  ) {
     return ObservableCreate<Feature>(async observer => {
-      const pafRecords = await this.setup(opts)
+      let pafRecords = await this.setup(opts)
+      const { config } = opts
+      if (config && readConfObject(config, 'colorBy') === 'meanQueryIdentity') {
+        pafRecords = getWeightedMeans(pafRecords)
+      }
       const assemblyNames = this.getAssemblyNames()
       const { assemblyName } = region
 

plugins/config/src/ConfigurationEditorWidget/components/CodeEditor.js

@@ -12,20 +12,21 @@ const fontSize = '12px'
 const fontFamily =
   'Consolas, "Andale Mono WT", "Andale Mono", "Lucida Console", "Lucida Sans Typewriter", "DejaVu Sans Mono", "Bitstream Vera Sans Mono", "Liberation Mono", "Nimbus Mono L", Monaco, "Courier New", Courier, monospace'
 
-const useStyles = makeStyles({
+const useStyles = makeStyles(theme => ({
   callbackEditor: {
     fontFamily,
     fontSize,
+    background: theme.palette.background.default,
     overflowX: 'auto',
     marginTop: '16px',
-    borderBottom: '1px solid rgba(0,0,0,0.42)',
+    border: '1px solid rgba(0,0,0,0.42)',
   },
   syntaxHighlighter: {
     margin: 0,
     fontFamily,
     fontSize,
   },
-})
+}))
 
 // eslint-disable-next-line react/prop-types
 export default function CodeEditor({ contents, setContents }) {

plugins/config/src/ConfigurationEditorWidget/components/SlotEditor.js

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from 'react'
 import { observer } from 'mobx-react'
 import { getPropertyMembers, getEnv } from 'mobx-state-tree'
-import { FileSelector } from '@jbrowse/core/ui'
+import { FileSelector, SanitizedHTML } from '@jbrowse/core/ui'
 import {
   getPropertyType,
   getSubType,
@@ -37,12 +37,27 @@ import CallbackEditor from './CallbackEditor'
 import ColorEditor from './ColorEditor'
 import JsonEditor from './JsonEditor'
 
+// adds ability to have html in helperText. note that FormHelperTextProps is
+// div because the default is p which does not like div children
+const MyTextField = props => {
+  // eslint-disable-next-line react/prop-types
+  const { helperText } = props
+  return (
+    <TextField
+      {...props}
+      helperText={<SanitizedHTML html={helperText} />}
+      FormHelperTextProps={{
+        component: 'div',
+      }}
+      fullWidth
+    />
+  )
+}
+
 const StringEditor = observer(({ slot }) => (
-  <TextField
+  <MyTextField
     label={slot.name}
-    // error={filterError}
     helperText={slot.description}
-    fullWidth
     value={slot.value}
     onChange={evt => slot.set(evt.target.value)}
   />
@@ -52,7 +67,6 @@ const TextEditor = observer(({ slot }) => (
   <TextField
     label={slot.name}
     helperText={slot.description}
-    fullWidth
     multiline
     value={slot.value}
     onChange={evt => slot.set(evt.target.value)}
@@ -268,7 +282,7 @@ const NumberEditor = observer(({ slot }) => {
     }
   }, [slot, val])
   return (
-    <TextField
+    <MyTextField
       label={slot.name}
       helperText={slot.description}
       value={val}
@@ -287,7 +301,7 @@ const IntegerEditor = observer(({ slot }) => {
     }
   }, [slot, val])
   return (
-    <TextField
+    <MyTextField
       label={slot.name}
       helperText={slot.description}
       value={val}
@@ -297,7 +311,7 @@ const IntegerEditor = observer(({ slot }) => {
   )
 })
 
-const booleanEditor = observer(({ slot }) => (
+const BooleanEditor = observer(({ slot }) => (
   <FormControl>
     <FormControlLabel
       label={slot.name}
@@ -312,28 +326,26 @@ const booleanEditor = observer(({ slot }) => (
   </FormControl>
 ))
 
-const stringEnumEditor = observer(({ slot, slotSchema }) => {
+const StringEnumEditor = observer(({ slot, slotSchema }) => {
   const p = getPropertyMembers(getSubType(slotSchema))
   const choices = getUnionSubTypes(
     getUnionSubTypes(getSubType(getPropertyType(p, 'value')))[1],
   ).map(t => t.value)
 
   return (
-    <TextField
+    <MyTextField
       value={slot.value}
       label={slot.name}
       select
-      // error={filterError}
       helperText={slot.description}
-      fullWidth
       onChange={evt => slot.set(evt.target.value)}
     >
       {choices.map(str => (
         <MenuItem key={str} value={str}>
           {str}
         </MenuItem>
       ))}
-    </TextField>
+    </MyTextField>
   )
 })
 
@@ -359,8 +371,8 @@ const valueComponents = {
   number: NumberEditor,
   integer: IntegerEditor,
   color: ColorEditor,
-  stringEnum: stringEnumEditor,
-  boolean: booleanEditor,
+  stringEnum: StringEnumEditor,
+  boolean: BooleanEditor,
   frozen: JsonEditor,
   configRelationships: JsonEditor,
 }