diff --git a/DESCRIPTION b/DESCRIPTION index 5708356..9c02f07 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: oolong Title: Create Validation Tests for Automated Content Analysis -Version: 0.6.1 +Version: 0.6.2 Authors@R: c(person(given = "Chung-hong", family = "Chan", role = c("aut", "cre"), email = "chainsawtiney@gmail.com", comment = c(ORCID = "0000-0002-6232-7530")), person(given = "Marius", family = "Sältzer", role = c("aut"), email = "msaeltze@mail.uni-mannheim.de", comment = c(ORCID = "0000-0002-8604-4666"))) @@ -19,7 +19,7 @@ Imports: digest, R6, quanteda (>= 3.0.0), - irr, + icr, ggplot2, cowplot, cli, diff --git a/NEWS.md b/NEWS.md index b213ab0..7312301 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ # oolong 0.6.2 (development) * Add content for MH +* Use `icr` for the calculation of Krippendorff's Alpha # oolong 0.6.1 diff --git a/R/oolong_summary.R b/R/oolong_summary.R index ce660cb..fa84eed 100644 --- a/R/oolong_summary.R +++ b/R/oolong_summary.R @@ -3,7 +3,7 @@ #' These functions print or plot a useful summary of the results from \code{\link{summarize_oolong}}. For details, please see the overview vignette: \code{vignette("overview", package = "oolong")} #' #' @section Summary: -#' +#' #' Print function displays the following information: #' \describe{ #' \item{Mean model precision}{(wi, wsi) Higher value indicates better topic interpretability} @@ -18,9 +18,9 @@ #' \item{Correlation (average answer)}{(gs) Pearson's correlation between average answer and target value} #' \item{Corrlation (content length)}{(gs) Pearson's correlation between content length and target value} #' } -#' +#' #' @section Diagnostic plot: -#' +#' #' Plot function displays a diagnostic plot with the following subplots (gs only). #' \describe{ #' \item{Top left}{Correlation between answer from coders and target value to check for correlation between two values. Both axes are minmax transformed.} @@ -28,7 +28,7 @@ #' \item{Bottom left}{Correlation between target value and content length to check for the influence of content length.} #' \item{Bottom right}{Cook's distance to check for influential observations.} #' } -#' +#' #' @param x an oolong_summary #' @param ... other parameters #' @method print oolong_summary @@ -53,7 +53,7 @@ plot.oolong_summary <- function(x, ...) { } else { .cstop(TRUE, "Don't know how to plot this oolong_summary.") } - + } .print_oolong_summary_tm <- function(oolong_summary) { @@ -85,7 +85,7 @@ plot.oolong_summary <- function(x, ...) { .print_oolong_summary_gs <- function(oolong_summary) { cli::cli_h1("Summary (gold standard generation):") - .cp(oolong_summary$n_models > 1, "Krippendorff's Alpha: ", round(oolong_summary$kripp_alpha$value, 3)) + .cp(oolong_summary$n_models > 1, "Krippendorff's Alpha: ", round(oolong_summary$kripp_alpha$alpha, 3)) .cp(!is.null(oolong_summary$cor), "Correlation: ", round(oolong_summary$cor$estimate, 3), " (p = ", round(oolong_summary$cor$p.value, 4), ")") .cp(!is.null(oolong_summary$cor_length), "Effect of content length: ", round(oolong_summary$cor_length$estimate, 3), " (p = ", round(oolong_summary$cor_length$p.value, 4), ")") } @@ -93,7 +93,7 @@ plot.oolong_summary <- function(x, ...) { #' Summarize oolong objects #' #' This function summarizes one or more oolong objects. All oolong objects must be locked. -#' +#' #' @param ... (tm/gs) one or more oolong objects to be summarized #' @param target_value (gs) a vector of numeric values, the value you want to validate against the human-coded gold standard. One example of this target value is sentiment score extracted automatically from text #' @param n_iter (ti) number of iterations to calculate the median test @@ -127,7 +127,7 @@ plot.oolong_summary <- function(x, ...) { #' @author Chung-hong Chan #' @references #' Chang, J., Gerrish, S., Wang, C., Boyd-Graber, J. L., & Blei, D. M. (2009). Reading tea leaves: How humans interpret topic models. In Advances in neural information processing systems (pp. 288-296). -#' +#' #' Song et al. (2020) In validations we trust? The impact of imperfect human annotations as a gold standard on the quality of validation of automated content analysis. Political Communication. #' #' Ying, L., Montgomery, J. M., & Stewart, B. M. (2021). Topics, Concepts, and Measurement: A Crowdsourced Procedure for Validating Topics as Measures. Political Analysis. diff --git a/R/oolong_summary_gs.R b/R/oolong_summary_gs.R index 714060a..8501bc5 100644 --- a/R/oolong_summary_gs.R +++ b/R/oolong_summary_gs.R @@ -1,5 +1,5 @@ .minmax <- function(x) { - (x - min(x)) / (max(x) - min(x)) + (x - min(x)) / (max(x) - min(x)) } .corr_plot <- function(answers, target_value) { @@ -37,7 +37,7 @@ avg_answer <- apply(answers, 1, mean) answers$avg_answer <- avg_answer if (length(obj_list) > 1) { - kripp <- irr::kripp.alpha(t(as.matrix(answers[,grepl("^answer", colnames(answers))])), method = "ordinal") + kripp <- icr::krippalpha(t(as.matrix(answers[,grepl("^answer", colnames(answers))])), metric = "ordinal") } else { kripp <- NA } diff --git a/R/oolong_summary_tm.R b/R/oolong_summary_tm.R index 8f82dcc..c49d3f6 100644 --- a/R/oolong_summary_tm.R +++ b/R/oolong_summary_tm.R @@ -52,7 +52,7 @@ res$multiple_test <- NA res$rater_precision_p_value <- NA } else { - res$kripp_alpha <- irr::kripp.alpha(t(ifelse(correction_matrix, 2, 1)))$value + res$kripp_alpha <- icr::krippalpha(t(ifelse(correction_matrix, 2, 1)))$alpha res$multiple_test <- purrr::map(n_correct, ~binom.test(., n = nrow(correction_matrix), p = 1/n_choices, alternative = "greater")) res$rater_precision_p_value <- .combine_p_fisher(purrr::map_dbl(res$multiple_test, "p.value")) } @@ -90,7 +90,7 @@ if (length(obj_list) == 1) { res$kripp_alpha_wsi <- NA } else { - res$kripp_alpha_wsi <- irr::kripp.alpha(t(ifelse(correction_matrix, 2, 1)))$value + res$kripp_alpha_wsi <- icr::krippalpha(t(ifelse(correction_matrix, 2, 1)))$alpha } } return(res) diff --git a/tests/testthat/_snaps/printing.md b/tests/testthat/_snaps/printing.md index c9d7967..8177bde 100644 --- a/tests/testthat/_snaps/printing.md +++ b/tests/testthat/_snaps/printing.md @@ -114,7 +114,7 @@ i Mean model precision: 0.833333333333333 i K Precision: 0.3, 0.7, 0.7, 0.7, 1, 1, 1, 1, 1, 1 - i Krippendorff's alpha: 0.056 + i Krippendorff's alpha: 0.072 ---