Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow designation of a custom name for the value_counts "count" column #16434

Merged
merged 2 commits into from
May 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions crates/polars-ops/src/series/ops/various.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,21 @@ use crate::series::ops::SeriesSealed;
pub trait SeriesMethods: SeriesSealed {
/// Create a [`DataFrame`] with the unique `values` of this [`Series`] and a column `"counts"`
/// with dtype [`IdxType`]
fn value_counts(&self, sort: bool, parallel: bool) -> PolarsResult<DataFrame> {
fn value_counts(&self, sort: bool, parallel: bool, name: String) -> PolarsResult<DataFrame> {
let s = self.as_series();
polars_ensure!(
s.name() != "count",
Duplicate: "using `value_counts` on a column named 'count' would lead to duplicate column names"
s.name() != name,
Duplicate: "using `value_counts` on a column/series named '{}' would lead to duplicate column names; change `name` to fix", name,
);
// we need to sort here as well in case of `maintain_order` because duplicates behavior is undefined
let groups = s.group_tuples(parallel, sort)?;
let values = unsafe { s.agg_first(&groups) };
let counts = groups.group_count().with_name("count");
let counts = groups.group_count().with_name(name.as_str());
let cols = vec![values, counts.into_series()];
let df = unsafe { DataFrame::new_no_checks(cols) };
if sort {
df.sort(
["count"],
[name],
SortMultipleOptions::default()
.with_order_descending(true)
.with_multithreaded(parallel),
Expand Down
9 changes: 7 additions & 2 deletions crates/polars-plan/src/dsl/function_expr/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,13 @@ pub(super) fn replace_time_zone(
}

#[cfg(feature = "dtype-struct")]
pub(super) fn value_counts(s: &Series, sort: bool, parallel: bool) -> PolarsResult<Series> {
s.value_counts(sort, parallel)
pub(super) fn value_counts(
s: &Series,
sort: bool,
parallel: bool,
name: String,
) -> PolarsResult<Series> {
s.value_counts(sort, parallel, name)
.map(|df| df.into_struct(s.name()).into_series())
}

Expand Down
14 changes: 12 additions & 2 deletions crates/polars-plan/src/dsl/function_expr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ pub enum FunctionExpr {
ValueCounts {
sort: bool,
parallel: bool,
name: String,
},
#[cfg(feature = "unique_counts")]
UniqueCounts,
Expand Down Expand Up @@ -463,9 +464,14 @@ impl Hash for FunctionExpr {
#[cfg(feature = "cum_agg")]
CumMax { reverse } => reverse.hash(state),
#[cfg(feature = "dtype-struct")]
ValueCounts { sort, parallel } => {
ValueCounts {
sort,
parallel,
name,
} => {
sort.hash(state);
parallel.hash(state);
name.hash(state);
},
#[cfg(feature = "unique_counts")]
UniqueCounts => {},
Expand Down Expand Up @@ -999,7 +1005,11 @@ impl From<FunctionExpr> for SpecialEq<Arc<dyn SeriesUdf>> {
#[cfg(feature = "cum_agg")]
CumMax { reverse } => map!(cum::cum_max, reverse),
#[cfg(feature = "dtype-struct")]
ValueCounts { sort, parallel } => map!(dispatch::value_counts, sort, parallel),
ValueCounts {
sort,
parallel,
name,
} => map!(dispatch::value_counts, sort, parallel, name.clone()),
#[cfg(feature = "unique_counts")]
UniqueCounts => map!(dispatch::unique_counts),
Reverse => map!(dispatch::reverse),
Expand Down
8 changes: 6 additions & 2 deletions crates/polars-plan/src/dsl/function_expr/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,14 @@ impl FunctionExpr {
#[cfg(feature = "top_k")]
TopKBy { .. } => mapper.with_same_dtype(),
#[cfg(feature = "dtype-struct")]
ValueCounts { .. } => mapper.map_dtype(|dt| {
ValueCounts {
sort: _,
parallel: _,
name,
} => mapper.map_dtype(|dt| {
DataType::Struct(vec![
Field::new(fields[0].name().as_str(), dt.clone()),
Field::new("count", IDX_DTYPE),
Field::new(name, IDX_DTYPE),
])
}),
#[cfg(feature = "unique_counts")]
Expand Down
16 changes: 10 additions & 6 deletions crates/polars-plan/src/dsl/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1714,12 +1714,16 @@ impl Expr {
#[cfg(feature = "dtype-struct")]
/// Count all unique values and create a struct mapping value to count.
/// (Note that it is better to turn parallel off in the aggregation context).
pub fn value_counts(self, sort: bool, parallel: bool) -> Self {
self.apply_private(FunctionExpr::ValueCounts { sort, parallel })
.with_function_options(|mut opts| {
opts.pass_name_to_apply = true;
opts
})
pub fn value_counts(self, sort: bool, parallel: bool, name: String) -> Self {
self.apply_private(FunctionExpr::ValueCounts {
sort,
parallel,
name,
})
.with_function_options(|mut opts| {
opts.pass_name_to_apply = true;
opts
})
}

#[cfg(feature = "unique_counts")]
Expand Down
4 changes: 2 additions & 2 deletions docs/src/rust/user-guide/expressions/structs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let out = ratings
.clone()
.lazy()
.select([col("Theatre").value_counts(true, true)])
.select([col("Theatre").value_counts(true, true, "count".to_string())])
.collect()?;
println!("{}", &out);
// --8<-- [end:state_value_counts]
Expand All @@ -26,7 +26,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let out = ratings
.clone()
.lazy()
.select([col("Theatre").value_counts(true, true)])
.select([col("Theatre").value_counts(true, true, "count".to_string())])
.unnest(["Theatre"])
.collect()?;
println!("{}", &out);
Expand Down
4 changes: 2 additions & 2 deletions py-polars/polars/dataframe/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -2975,7 +2975,7 @@ def write_excel(
... ws.write(len(df) + 6, 1, "Customised conditional formatting", fmt_title)

Export a table containing two different types of sparklines. Use default
options for the "trend" sparkline and customised options (and positioning)
options for the "trend" sparkline and customized options (and positioning)
for the "+/-" win_loss sparkline, with non-default integer dtype formatting,
column totals, a subtle two-tone heatmap and hidden worksheet gridlines:

Expand All @@ -2995,7 +2995,7 @@ def write_excel(
... sparklines={
... # default options; just provide source cols
... "trend": ["q1", "q2", "q3", "q4"],
... # customised sparkline type, with positioning directive
... # customized sparkline type, with positioning directive
... "+/-": {
... "columns": ["q1", "q2", "q3", "q4"],
... "insert_after": "id",
Expand Down
27 changes: 22 additions & 5 deletions py-polars/polars/expr/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -10909,7 +10909,9 @@ def extend_constant(self, value: IntoExpr, n: int | IntoExprColumn) -> Self:
return self._from_pyexpr(self._pyexpr.extend_constant(value, n))

@deprecate_renamed_parameter("multithreaded", "parallel", version="0.19.0")
def value_counts(self, *, sort: bool = False, parallel: bool = False) -> Self:
def value_counts(
self, *, sort: bool = False, parallel: bool = False, name: str = "count"
) -> Self:
"""
Count the occurrences of unique values.

Expand All @@ -10924,6 +10926,8 @@ def value_counts(self, *, sort: bool = False, parallel: bool = False) -> Self:
.. note::
This option should likely not be enabled in a group by context,
as the computation is already parallelized per group.
name
Give the resulting count field a specific name; defaults to "count".

Returns
-------
Expand All @@ -10948,9 +10952,10 @@ def value_counts(self, *, sort: bool = False, parallel: bool = False) -> Self:
β”‚ {"blue",3} β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜

Sort the output by count.
Sort the output by (descending) count and customize the count field name.

>>> df.select(pl.col("color").value_counts(sort=True))
>>> df = df.select(pl.col("color").value_counts(sort=True, name="n"))
>>> df
shape: (3, 1)
β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
β”‚ color β”‚
Expand All @@ -10961,8 +10966,20 @@ def value_counts(self, *, sort: bool = False, parallel: bool = False) -> Self:
β”‚ {"red",2} β”‚
β”‚ {"green",1} β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
"""
return self._from_pyexpr(self._pyexpr.value_counts(sort, parallel))

>>> df.unnest("color")
shape: (3, 2)
β”Œβ”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”
β”‚ color ┆ n β”‚
β”‚ --- ┆ --- β”‚
β”‚ str ┆ u32 β”‚
β•žβ•β•β•β•β•β•β•β•ͺ═════║
β”‚ blue ┆ 3 β”‚
β”‚ red ┆ 2 β”‚
β”‚ green ┆ 1 β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”˜
"""
return self._from_pyexpr(self._pyexpr.value_counts(sort, parallel, name))

def unique_counts(self) -> Self:
"""
Expand Down
30 changes: 17 additions & 13 deletions py-polars/polars/series/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -2762,7 +2762,9 @@ def hist(
else:
return out.struct.unnest()

def value_counts(self, *, sort: bool = False, parallel: bool = False) -> DataFrame:
def value_counts(
self, *, sort: bool = False, parallel: bool = False, name: str = "count"
) -> DataFrame:
"""
Count the occurrences of unique values.

Expand All @@ -2777,6 +2779,8 @@ def value_counts(self, *, sort: bool = False, parallel: bool = False) -> DataFra
.. note::
This option should likely not be enabled in a group by context,
as the computation is already parallelized per group.
name
Give the resulting count column a specific name; defaults to "count".

Returns
-------
Expand All @@ -2798,22 +2802,22 @@ def value_counts(self, *, sort: bool = False, parallel: bool = False) -> DataFra
β”‚ blue ┆ 3 β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜

Sort the output by count.
Sort the output by count and customize the count column name.

>>> s.value_counts(sort=True)
>>> s.value_counts(sort=True, name="n")
shape: (3, 2)
β”Œβ”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”
β”‚ color ┆ count β”‚
β”‚ --- ┆ --- β”‚
β”‚ str ┆ u32 β”‚
β•žβ•β•β•β•β•β•β•β•ͺ═══════║
β”‚ blue ┆ 3 β”‚
β”‚ red ┆ 2 β”‚
β”‚ green ┆ 1 β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”˜
β”Œβ”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”
β”‚ color ┆ n β”‚
β”‚ --- ┆ --- β”‚
β”‚ str ┆ u32 β”‚
β•žβ•β•β•β•β•β•β•β•ͺ═════║
β”‚ blue ┆ 3 β”‚
β”‚ red ┆ 2 β”‚
β”‚ green ┆ 1 β”‚
β””β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”˜
"""
return pl.DataFrame._from_pydf(
self._s.value_counts(sort=sort, parallel=parallel)
self._s.value_counts(sort=sort, parallel=parallel, name=name)
)

def unique_counts(self) -> Series:
Expand Down
4 changes: 2 additions & 2 deletions py-polars/src/expr/general.rs
Original file line number Diff line number Diff line change
Expand Up @@ -250,8 +250,8 @@ impl PyExpr {
fn len(&self) -> Self {
self.inner.clone().len().into()
}
fn value_counts(&self, sort: bool, parallel: bool) -> Self {
self.inner.clone().value_counts(sort, parallel).into()
fn value_counts(&self, sort: bool, parallel: bool, name: String) -> Self {
self.inner.clone().value_counts(sort, parallel, name).into()
}
fn unique_counts(&self) -> Self {
self.inner.clone().unique_counts().into()
Expand Down
1 change: 1 addition & 0 deletions py-polars/src/lazyframe/visitor/expr_nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -957,6 +957,7 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult<PyObject> {
FunctionExpr::ValueCounts {
sort: _,
parallel: _,
name: _,
} => return Err(PyNotImplementedError::new_err("value counts")),
FunctionExpr::UniqueCounts => {
return Err(PyNotImplementedError::new_err("unique counts"))
Expand Down
4 changes: 2 additions & 2 deletions py-polars/src/series/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -732,10 +732,10 @@ impl PySeries {
self.series.tail(Some(n)).into()
}

fn value_counts(&self, sort: bool, parallel: bool) -> PyResult<PyDataFrame> {
fn value_counts(&self, sort: bool, parallel: bool, name: String) -> PyResult<PyDataFrame> {
let out = self
.series
.value_counts(sort, parallel)
.value_counts(sort, parallel, name)
.map_err(PyPolarsErr::from)?;
Ok(out.into())
}
Expand Down
4 changes: 2 additions & 2 deletions py-polars/tests/unit/io/test_spreadsheet.py
Original file line number Diff line number Diff line change
Expand Up @@ -526,7 +526,7 @@ def test_read_excel_all_sheets_with_sheet_name(path_xlsx: Path, engine: str) ->
"column_totals": True,
"float_precision": 0,
},
# slightly customised formatting, with some formulas
# slightly customized formatting, with some formulas
{
"position": (0, 0),
"table_style": {
Expand Down Expand Up @@ -555,7 +555,7 @@ def test_read_excel_all_sheets_with_sheet_name(path_xlsx: Path, engine: str) ->
"column_totals": True,
"row_totals": True,
},
# heavily customised formatting/definition
# heavily customized formatting/definition
{
"position": "A1",
"table_name": "PolarsFrameData",
Expand Down
19 changes: 15 additions & 4 deletions py-polars/tests/unit/operations/test_value_counts.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,20 @@ def test_value_counts_expr() -> None:


def test_value_counts_duplicate_name() -> None:
s = pl.Series("count", [1])
s = pl.Series("count", [1, 0, 1])

with pytest.raises(pl.DuplicateError, match="count"):
# default name is 'count' ...
with pytest.raises(
pl.DuplicateError,
match="duplicate column names; change `name` to fix",
):
s.value_counts()

def test_count() -> None:
assert pl.Series([None, 1, None, 2, 3]).count() == 3
# ... but can customize that
assert_frame_equal(
pl.DataFrame({"count": [1, 0], "n": [2, 1]}, schema_overrides={"n": pl.UInt32}),
s.value_counts(name="n", sort=True),
)

df = pl.DataFrame({"a": [None, 1, None, 2, 3]})
assert df.select(pl.col("a").count()).item() == 3
Expand All @@ -66,3 +73,7 @@ def test_count() -> None:
"literal": [1],
"a": [3],
}


def test_count() -> None:
assert pl.Series([None, 1, None, 2, 3]).count() == 3