Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Discrete normal #216

Merged
merged 4 commits into from
Dec 1, 2023
+116 −35
Merged

Discrete normal #216

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9cf6d2a
Add DiscreteNormal and DiscreteTruncateNormal
vankesteren Nov 29, 2023
f3115b8
Minor fixes to make tests & CI happy
vankesteren Nov 29, 2023
02048fe
Fix ruff issue
qubixes Nov 30, 2023
babb0e4
Add tests
qubixes Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 14 additions & 26 deletions examples/example_gmf_titanic.json
View file
Original file line number Diff line number Diff line change
@@ -4,9 +4,9 @@
"provenance": {
"created by": {
"name": "metasyn",
"version": "0.6.1.dev32+g871b8ec"
"version": "0.6.1.dev35+gf3115b8.d20231130"
},
"creation time": "2023-11-21T13:22:03.439633"
"creation time": "2023-11-30T09:40:01.749592"
},
"vars": [
{
@@ -70,13 +70,15 @@
"dtype": "Int64",
"prop_missing": 0.19865319865319866,
"distribution": {
"implements": "core.discrete_uniform",
"implements": "core.discrete_truncated_normal",
"version": "1.0",
"provenance": "builtin",
"class_name": "DiscreteUniformDistribution",
"class_name": "DiscreteTruncatedNormalDistribution",
"parameters": {
"low": 0,
"high": 81
"lower_bound": -1e-08,
"upper_bound": 80.00000001,
"mu": 28.403638823278087,
"sigma": 15.862325051407092
}
}
},
@@ -86,29 +88,15 @@
"dtype": "Int64",
"prop_missing": 0.0,
"distribution": {
"implements": "core.multinoulli",
"implements": "core.discrete_truncated_normal",
"version": "1.0",
"provenance": "builtin",
"class_name": "MultinoulliDistribution",
"class_name": "DiscreteTruncatedNormalDistribution",
"parameters": {
"labels": [
0,
1,
2,
3,
4,
5,
6
],
"probs": [
0.760942760942761,
0.13243546576879914,
0.08978675645342313,
0.0056116722783389455,
0.004489337822671157,
0.0056116722783389455,
0.0011223344556677893
]
"lower_bound": -1e-08,
"upper_bound": 6.00000001,
"mu": -380.6440825743838,
"sigma": 12.066012048277289
}
}
},
29 changes: 22 additions & 7 deletions metasyn/distribution/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -19,6 +19,8 @@
UniformTimeDistribution,
)
from metasyn.distribution.discrete import (
DiscreteNormalDistribution,
DiscreteTruncatedNormalDistribution,
DiscreteUniformDistribution,
PoissonDistribution,
UniqueKeyDistribution,
@@ -32,11 +34,24 @@
from metasyn.distribution.regex import RegexDistribution, UniqueRegexDistribution

__all__ = [
"MultinoulliDistribution", "UniformDistribution", "NormalDistribution",
"LogNormalDistribution", "TruncatedNormalDistribution", "ExponentialDistribution",
"DiscreteUniformDistribution", "PoissonDistribution", "UniqueKeyDistribution",
"UniformDateDistribution", "UniformDateTimeDistribution", "UniformTimeDistribution",
"FakerDistribution", "UniqueFakerDistribution", "RegexDistribution", "UniqueRegexDistribution",
"NADistribution", "FreeTextDistribution"

"MultinoulliDistribution",
"ExponentialDistribution",
"LogNormalDistribution",
"NormalDistribution",
"TruncatedNormalDistribution",
"UniformDistribution",
"UniformDateDistribution",
"UniformDateTimeDistribution",
"UniformTimeDistribution",
"DiscreteNormalDistribution",
"DiscreteTruncatedNormalDistribution",
"DiscreteUniformDistribution",
"PoissonDistribution",
"UniqueKeyDistribution",
"FakerDistribution",
"FreeTextDistribution",
"UniqueFakerDistribution",
"NADistribution",
"RegexDistribution",
"UniqueRegexDistribution",
]
39 changes: 39 additions & 0 deletions metasyn/distribution/discrete.py
View file
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
from scipy.stats import poisson, randint

from metasyn.distribution.base import ScipyDistribution, metadist
from metasyn.distribution.continuous import NormalDistribution, TruncatedNormalDistribution


@metadist(implements="core.discrete_uniform", var_type="discrete")
@@ -48,6 +49,44 @@ def _param_schema(cls):
"high": {"type": "integer"},
}

@metadist(implements="core.discrete_normal", var_type="discrete")
class DiscreteNormalDistribution(NormalDistribution):
"""Normal distribution for integer type.

This class implements the normal/gaussian distribution and takes
the average and standard deviation as initialization input.

Parameters
----------
mean: float
Mean of the normal distribution.

std_dev: float
Standard deviation of the normal distribution.
"""

def draw(self):
return int(super().draw())

@metadist(implements="core.discrete_truncated_normal", var_type="discrete")
class DiscreteTruncatedNormalDistribution(TruncatedNormalDistribution):
"""Truncated normal distribution for integer type.

Parameters
----------
lower_bound: float
Lower bound of the truncated normal distribution.
upper_bound: float
Upper bound of the truncated normal distribution.
mu: float
Mean of the non-truncated normal distribution.
sigma: float
Standard deviation of the non-truncated normal distribution.
"""

def draw(self):
return int(super().draw())


@metadist(implements="core.poisson", var_type="discrete")
class PoissonDistribution(ScipyDistribution):
3 changes: 3 additions & 0 deletions metasyn/provider.py
View file
Original file line number Diff line number Diff line change
@@ -35,6 +35,8 @@
UniformTimeDistribution,
)
from metasyn.distribution.discrete import (
DiscreteNormalDistribution,
DiscreteTruncatedNormalDistribution,
DiscreteUniformDistribution,
PoissonDistribution,
UniqueKeyDistribution,
@@ -114,6 +116,7 @@ class BuiltinDistributionProvider(BaseDistributionProvider):
name = "builtin"
version = "1.1"
distributions = [
DiscreteNormalDistribution, DiscreteTruncatedNormalDistribution,
DiscreteUniformDistribution, PoissonDistribution, UniqueKeyDistribution,
UniformDistribution, NormalDistribution, LogNormalDistribution,
TruncatedNormalDistribution, ExponentialDistribution,
40 changes: 38 additions & 2 deletions tests/test_discrete.py
View file
Original file line number Diff line number Diff line change
@@ -4,9 +4,11 @@
import pandas as pd
import polars as pl
from pytest import mark
from scipy.stats import poisson
from scipy import stats

from metasyn.distribution.discrete import (
DiscreteNormalDistribution,
DiscreteTruncatedNormalDistribution,
DiscreteUniformDistribution,
PoissonDistribution,
UniqueKeyDistribution,
@@ -64,8 +66,42 @@ def test_integer_key(data, better_than_uniform, consecutive, series_type):

@mark.parametrize("series_type", [pd.Series, pl.Series])
def test_poisson(series_type):
series = series_type(poisson(mu=10).rvs(1000))
series = series_type(stats.poisson(mu=10).rvs(1000))
dist = PoissonDistribution.fit(series)
dist_unif = DiscreteUniformDistribution.fit(series)
assert fabs(dist.mu - 10) < 1
assert dist.information_criterion(series) < dist_unif.information_criterion(series)

@mark.parametrize(
"lower_bound,upper_bound,mu,sigma",
[
(-1, 1, 0, 0.5),
(-10, -8, 0, 5),
(-5, 3, 2, 2),
]
)
def test_trunc_normal(lower_bound, upper_bound, mu, sigma):
a, b = (lower_bound-mu)/sigma, (upper_bound-mu)/sigma
values = pl.Series(stats.truncnorm(a=a, b=b, loc=mu, scale=sigma).rvs(5000)).cast(pl.Int64)
dist = DiscreteTruncatedNormalDistribution.fit(values)
dist_uniform = DiscreteUniformDistribution.fit(values)
assert dist.information_criterion(values) < dist_uniform.information_criterion(values)
assert isinstance(dist.draw(), int)


@mark.parametrize(
"mean,std_dev",
[
(0, 1),
(-10, 2),
(100, 200),
]
)
def test_normal(mean, std_dev):
values = pl.Series(stats.norm(loc=mean, scale=std_dev).rvs(1000)).cast(pl.Int64)
dist = DiscreteNormalDistribution.fit(values)
dist_uniform = DiscreteUniformDistribution.fit(values)
assert dist.information_criterion(values) < dist_uniform.information_criterion(values)
assert (dist.mean - mean)/std_dev < 0.5
assert (dist.std_dev - std_dev)/std_dev < 0.5
assert isinstance(dist.draw(), int)