From 117ff02e8fdc0f290dcf2ab9a05293f9a41c58a4 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 08:02:30 -0400 Subject: [PATCH 1/9] update to compute AP as trec_eval does. --- db/scorers/ap@10.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index 9861a24d0..a830a265b 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -3,10 +3,23 @@ total = 0; // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k - +// for each returned document, calculate precision each time a new +// relevant document is added to the ranked list. +var count = 0; eachDoc(function(doc, i) { - total += avgRating(i+1) + if (hasDocRating(i) && (docRating(i)) > 0) { + count++; + total += count/(i+1) + } }, k); - -var score = total / k; +// count up the total number of relevant (not judged) documents +var totalRel = 0; +for (var i = 0; i < bestDocs.length; i++) { + if (bestDocs[i].rating > 0) { + totalRel++; + } +} +// AP is the sum of the precision points divided by the total +// number of relevant documents +var score = total / totalRel; setScore(score); From 4e16197c8a5a7a0a32944f3bf376270551ae28e6 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 08:08:04 -0400 Subject: [PATCH 2/9] update example to use the AP implementation. --- .../javascripts/factories/ScorerFactory.js | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 538b56ab0..22c141c3c 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -11,16 +11,30 @@ var Scorer = function(data) { var self = this; var defaultAlgorithm = [ - '// This is the AP@10 formula as an example', - 'var k = 10; // @Rank', - 'total = 0;', - '// if less than K results, need to reduce K now or final score is too low', - 'k = numReturned() < k ? numReturned() : k', - '', - 'eachDoc(function(doc, i) {', - ' total += avgRating(i+1)', - '}, k);', - 'var score = total / k;', + '// This is the AP@10 formula as an example', + 'var k = 10; // @Rank', + 'total = 0;' + '// if less than K results, need to reduce K now or final score is too low' + 'k = numReturned() < k ? numReturned() : k', + '// for each returned document, calculate precision each time a new' + '// relevant document is added to the ranked list.' + 'var count = 0;' + 'eachDoc(function(doc, i) {', + 'if (hasDocRating(i) && (docRating(i)) > 0) {', + 'count++;', + 'total += count/(i+1)', + '}', + '}, k);', + '// count up the total number of relevant (not judged) documents', + 'var totalRel = 0;', + 'for (var i = 0; i < bestDocs.length; i++) {', + 'if (bestDocs[i].rating > 0) {', + 'totalRel++;', + '}', + '}', + '// AP is the sum of the precision points divided by the total', + '// number of relevant documents', + 'var score = total / totalRel;', 'setScore(score);', ].join('\n'); From b8d20a35fb649d383bb7219c97f5e43c9c5ed9fc Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 09:10:47 -0400 Subject: [PATCH 3/9] update to address javascript test issues. --- app/assets/javascripts/factories/ScorerFactory.js | 8 ++++---- db/scorers/ap@10.js | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 22c141c3c..cf509acf6 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -13,12 +13,13 @@ var defaultAlgorithm = [ '// This is the AP@10 formula as an example', 'var k = 10; // @Rank', + 'var count = 0;', + 'var totalRel = 0;' 'total = 0;' '// if less than K results, need to reduce K now or final score is too low' 'k = numReturned() < k ? numReturned() : k', '// for each returned document, calculate precision each time a new' '// relevant document is added to the ranked list.' - 'var count = 0;' 'eachDoc(function(doc, i) {', 'if (hasDocRating(i) && (docRating(i)) > 0) {', 'count++;', @@ -26,15 +27,14 @@ '}', '}, k);', '// count up the total number of relevant (not judged) documents', - 'var totalRel = 0;', - 'for (var i = 0; i < bestDocs.length; i++) {', + 'for (let i = 0; i < bestDocs.length; i++) {', 'if (bestDocs[i].rating > 0) {', 'totalRel++;', '}', '}', '// AP is the sum of the precision points divided by the total', '// number of relevant documents', - 'var score = total / totalRel;', + 'let score = total / totalRel;', 'setScore(score);', ].join('\n'); diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index a830a265b..2434d1fba 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -1,11 +1,11 @@ var k = 10; // @Rank +var count = 0; +var totalRel = 0; total = 0; - // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k // for each returned document, calculate precision each time a new // relevant document is added to the ranked list. -var count = 0; eachDoc(function(doc, i) { if (hasDocRating(i) && (docRating(i)) > 0) { count++; @@ -13,13 +13,12 @@ eachDoc(function(doc, i) { } }, k); // count up the total number of relevant (not judged) documents -var totalRel = 0; -for (var i = 0; i < bestDocs.length; i++) { +for (let i = 0; i < bestDocs.length; i++) { if (bestDocs[i].rating > 0) { totalRel++; } } // AP is the sum of the precision points divided by the total // number of relevant documents -var score = total / totalRel; +let score = total / totalRel; setScore(score); From 9c02b536d905bd9f38819ede22a2139e0d6d1efa Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 09:14:46 -0400 Subject: [PATCH 4/9] update to address javascript test issues. --- app/assets/javascripts/factories/ScorerFactory.js | 8 ++++---- db/scorers/ap@10.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index cf509acf6..2330c576d 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -12,9 +12,9 @@ var self = this; var defaultAlgorithm = [ '// This is the AP@10 formula as an example', - 'var k = 10; // @Rank', - 'var count = 0;', - 'var totalRel = 0;' + 'let k = 10; // @Rank', + 'let count = 0;', + 'let totalRel = 0;' 'total = 0;' '// if less than K results, need to reduce K now or final score is too low' 'k = numReturned() < k ? numReturned() : k', @@ -34,7 +34,7 @@ '}', '// AP is the sum of the precision points divided by the total', '// number of relevant documents', - 'let score = total / totalRel;', + 'const score = total / totalRel;', 'setScore(score);', ].join('\n'); diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index 2434d1fba..fc3daddd1 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -1,6 +1,6 @@ -var k = 10; // @Rank -var count = 0; -var totalRel = 0; +let k = 10; // @Rank +let count = 0; +let totalRel = 0; total = 0; // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k @@ -20,5 +20,5 @@ for (let i = 0; i < bestDocs.length; i++) { } // AP is the sum of the precision points divided by the total // number of relevant documents -let score = total / totalRel; +const score = total / totalRel; setScore(score); From d77868d4462303197696cce1c256c4b207cb511d Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 09:18:01 -0400 Subject: [PATCH 5/9] update to address javascript test issues. --- app/assets/javascripts/factories/ScorerFactory.js | 2 +- db/scorers/ap@10.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 2330c576d..cc3d107d8 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -15,7 +15,7 @@ 'let k = 10; // @Rank', 'let count = 0;', 'let totalRel = 0;' - 'total = 0;' + 'let total = 0;' '// if less than K results, need to reduce K now or final score is too low' 'k = numReturned() < k ? numReturned() : k', '// for each returned document, calculate precision each time a new' diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index fc3daddd1..af07ccf12 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -1,7 +1,7 @@ let k = 10; // @Rank let count = 0; let totalRel = 0; -total = 0; +let total = 0; // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k // for each returned document, calculate precision each time a new From 9a868d798cb6cd2bfe18ce0104280030a84ffc41 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 09:19:14 -0400 Subject: [PATCH 6/9] update to address javascript test issues. --- app/assets/javascripts/factories/ScorerFactory.js | 10 +++++----- db/scorers/ap@10.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index cc3d107d8..31d02c1ad 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -14,12 +14,12 @@ '// This is the AP@10 formula as an example', 'let k = 10; // @Rank', 'let count = 0;', - 'let totalRel = 0;' - 'let total = 0;' - '// if less than K results, need to reduce K now or final score is too low' + 'let totalRel = 0;', + 'total = 0;', + '// if less than K results, need to reduce K now or final score is too low', 'k = numReturned() < k ? numReturned() : k', - '// for each returned document, calculate precision each time a new' - '// relevant document is added to the ranked list.' + '// for each returned document, calculate precision each time a new', + '// relevant document is added to the ranked list.', 'eachDoc(function(doc, i) {', 'if (hasDocRating(i) && (docRating(i)) > 0) {', 'count++;', diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index af07ccf12..fc3daddd1 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -1,7 +1,7 @@ let k = 10; // @Rank let count = 0; let totalRel = 0; -let total = 0; +total = 0; // if less than K results, need to reduce K now or final score is too low k = numReturned() < k ? numReturned() : k // for each returned document, calculate precision each time a new From 11cb2e67172d2e86ea3d2b8db1e073099f1b0526 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 09:32:14 -0400 Subject: [PATCH 7/9] update to address javascript test issues. --- app/assets/javascripts/factories/ScorerFactory.js | 6 +++--- db/scorers/ap@10.js | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 31d02c1ad..ef3a87430 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -27,11 +27,11 @@ '}', '}, k);', '// count up the total number of relevant (not judged) documents', - 'for (let i = 0; i < bestDocs.length; i++) {', - 'if (bestDocs[i].rating > 0) {', + 'eachRatedDoc(function(doc, rating) {', + 'if (rating > 0) {', 'totalRel++;', '}', - '}', + '}, bestDocs.length);', '// AP is the sum of the precision points divided by the total', '// number of relevant documents', 'const score = total / totalRel;', diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index fc3daddd1..4a54f092a 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -13,11 +13,12 @@ eachDoc(function(doc, i) { } }, k); // count up the total number of relevant (not judged) documents -for (let i = 0; i < bestDocs.length; i++) { - if (bestDocs[i].rating > 0) { +eachRatedDoc(function(doc, rating) { + if (rating > 0) { totalRel++; } -} +}, bestDocs.length); + // AP is the sum of the precision points divided by the total // number of relevant documents const score = total / totalRel; From da59ed34eb7f85f1780bc32dd95ffbea29381e42 Mon Sep 17 00:00:00 2001 From: David Fisher Date: Wed, 25 May 2022 13:41:58 -0400 Subject: [PATCH 8/9] update to use the other iterator over judged documents. --- app/assets/javascripts/factories/ScorerFactory.js | 4 ++-- db/scorers/ap@10.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index ef3a87430..ed0cfdf2d 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -27,8 +27,8 @@ '}', '}, k);', '// count up the total number of relevant (not judged) documents', - 'eachRatedDoc(function(doc, rating) {', - 'if (rating > 0) {', + 'eachDocWithRating(function(doc) {', + 'if (doc.rating > 0) {', 'totalRel++;', '}', '}, bestDocs.length);', diff --git a/db/scorers/ap@10.js b/db/scorers/ap@10.js index 4a54f092a..ae84b5574 100644 --- a/db/scorers/ap@10.js +++ b/db/scorers/ap@10.js @@ -13,8 +13,8 @@ eachDoc(function(doc, i) { } }, k); // count up the total number of relevant (not judged) documents -eachRatedDoc(function(doc, rating) { - if (rating > 0) { +eachDocWithRating(function(doc) { + if (doc.rating > 0) { totalRel++; } }, bestDocs.length); From 413589d5cb7cc95dbca443f9229c9b4fc643491b Mon Sep 17 00:00:00 2001 From: "epugh@opensourceconnections.com" Date: Mon, 18 Jul 2022 12:44:59 -0400 Subject: [PATCH 9/9] um, missed this --- .../javascripts/factories/ScorerFactory.js | 34 +------------------ .../angular/services/scorerFactory_spec.js | 2 +- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/app/assets/javascripts/factories/ScorerFactory.js b/app/assets/javascripts/factories/ScorerFactory.js index 2b6bf5f9e..05f508942 100644 --- a/app/assets/javascripts/factories/ScorerFactory.js +++ b/app/assets/javascripts/factories/ScorerFactory.js @@ -10,42 +10,11 @@ function ScorerFactory($q, $timeout) { var Scorer = function(data) { var self = this; - var defaultAlgorithm = [ - '// This is the AP@10 formula as an example', - 'let k = 10; // @Rank', - 'let count = 0;', - 'let totalRel = 0;', - 'total = 0;', - '// if less than K results, need to reduce K now or final score is too low', - 'k = numReturned() < k ? numReturned() : k', - '// for each returned document, calculate precision each time a new', - '// relevant document is added to the ranked list.', - 'eachDoc(function(doc, i) {', - 'if (hasDocRating(i) && (docRating(i)) > 0) {', - 'count++;', - 'total += count/(i+1)', - '}', - '}, k);', - '// count up the total number of relevant (not judged) documents', - 'eachDocWithRating(function(doc) {', - 'if (doc.rating > 0) {', - 'totalRel++;', - '}', - '}, bestDocs.length);', - '// AP is the sum of the precision points divided by the total', - '// number of relevant documents', - 'const score = total / totalRel;', - 'setScore(score);', - ].join('\n'); if (angular.isUndefined(data)) { data = {}; } - if ( angular.isUndefined(data.code) ) { - data.code = defaultAlgorithm; - } - if ( angular.isUndefined(data.scale) ) { data.scale = ['0', '1']; data.scaleWithLabels = scaleToScaleWithLabels(data.scale, null); @@ -54,7 +23,6 @@ // Attributes self.code = data.code; self.colors = scaleToColors(data.scale); - self.defaultAlgorithm = defaultAlgorithm; self.displayName = setDisplayName(data.name, data.communal); self.error = false; self.manualMaxScore = data.manualMaxScore || false; @@ -92,7 +60,7 @@ self.getBestRatings = getBestRatings; - var DEFAULT_NUM_DOCS = 10; + const DEFAULT_NUM_DOCS = 10; // public functions diff --git a/spec/javascripts/angular/services/scorerFactory_spec.js b/spec/javascripts/angular/services/scorerFactory_spec.js index 09f5d5310..0dbee85bb 100644 --- a/spec/javascripts/angular/services/scorerFactory_spec.js +++ b/spec/javascripts/angular/services/scorerFactory_spec.js @@ -14,7 +14,7 @@ describe('Service: ScorerFactory', function () { var mockScorer = { 'scorerId': 1, 'name': 'Scorer 1', - 'code': scorerSvc.defaultAlgorithm, + 'code': "setScore(99)", 'scale': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 'owner_id': 1 };