diff --git a/dag/energy.yml b/dag/energy.yml index 0ec3bdb2621..f9d129f15d5 100644 --- a/dag/energy.yml +++ b/dag/energy.yml @@ -228,3 +228,54 @@ steps: # data://grapher/energy/2024-11-15/photovoltaic_cost_and_capacity: - data://garden/energy/2024-11-15/photovoltaic_cost_and_capacity + # + # Eurostat - Energy statistics, prices of natural gas and electricity + # + data://meadow/eurostat/2024-11-05/gas_and_electricity_prices: + - snapshot://eurostat/2024-11-05/gas_and_electricity_prices.zip + # + # Eurostat - Energy statistics, prices of natural gas and electricity + # + data://garden/eurostat/2024-11-05/gas_and_electricity_prices: + - data://meadow/eurostat/2024-11-05/gas_and_electricity_prices + # + # Ember - European wholesale electricity prices + # + data://meadow/ember/2024-11-20/european_wholesale_electricity_prices: + - snapshot://ember/2024-11-20/european_wholesale_electricity_prices.csv + # + # Ember - European wholesale electricity prices + # + data://garden/ember/2024-11-20/european_wholesale_electricity_prices: + - data://meadow/ember/2024-11-20/european_wholesale_electricity_prices + # + # IEA - Fossil fuel subsidies + # + data://meadow/iea/2024-11-20/fossil_fuel_subsidies: + - snapshot://iea/2024-11-20/fossil_fuel_subsidies.xlsx + # + # IEA - Fossil fuel subsidies + # + data://garden/iea/2024-11-20/fossil_fuel_subsidies: + - data://meadow/iea/2024-11-20/fossil_fuel_subsidies + # + # IEA - Fossil fuel subsidies + # + data://grapher/iea/2024-11-20/fossil_fuel_subsidies: + - data://garden/iea/2024-11-20/fossil_fuel_subsidies + # + # Energy prices + # + data://garden/energy/2024-11-20/energy_prices: + - data://garden/eurostat/2024-11-05/gas_and_electricity_prices + - data://garden/ember/2024-11-20/european_wholesale_electricity_prices + # + # Energy prices + # + data://grapher/energy/2024-11-20/energy_prices: + - data://garden/energy/2024-11-20/energy_prices + # + # Energy prices explorer + # + export://multidim/energy/latest/energy_prices: + - data://grapher/energy/2024-11-20/energy_prices diff --git a/docs/guides/data-work/export-data.md b/docs/guides/data-work/export-data.md index a3ff3341601..8dbe49b9ae2 100644 --- a/docs/guides/data-work/export-data.md +++ b/docs/guides/data-work/export-data.md @@ -31,101 +31,93 @@ ds_explorer.save() Multi-dimensional indicators are powered by a configuration that is typically created from a YAML file. The structure of the YAML file looks like this: -```yaml title="etl/steps/export/multidim/covid/latest/covid.deaths.yaml" -definitions: - table: {definitions.table} - +```yaml title="etl/steps/export/multidim/energy/latest/energy_prices.yaml" title: - title: COVID-19 deaths - titleVariant: by interval + title: "Energy prices" + titleVariant: "by energy source" defaultSelection: - - World - - Europe - - Asia + - "European Union (27)" topicTags: - - COVID-19 - + - "Energy" dimensions: - - slug: interval - name: Interval + - slug: "frequency" + name: "Frequency" choices: - - slug: weekly - name: Weekly - description: null - - slug: biweekly - name: Biweekly - description: null - - - slug: metric - name: Metric + - slug: "annual" + name: "Annual" + description: "Annual data" + - slug: "monthly" + name: "Monthly" + description: "Monthly data" + - slug: "source" + name: "Energy source" choices: - - slug: absolute - name: Absolute - description: null - - slug: per_capita - name: Per million people - description: null - - slug: change - name: Change from previous interval - description: null - + - slug: "electricity" + name: "Electricity" + - slug: "gas" + name: "Gas" + - slug: "unit" + name: "Unit" + choices: + - slug: "euro" + name: "Euro" + description: "Price in euros" + - slug: "pps" + name: "PPS" + description: "Price in Purchasing Power Standard" views: - - dimensions: - interval: weekly - metric: absolute - indicators: - y: "{definitions.table}#weekly_deaths" - - dimensions: - interval: weekly - metric: per_capita - indicators: - y: "{definitions.table}#weekly_deaths_per_million" - - dimensions: - interval: weekly - metric: change - indicators: - y: "{definitions.table}#weekly_pct_growth_deaths" - - - dimensions: - interval: biweekly - metric: absolute - indicators: - y: "{definitions.table}#biweekly_deaths" - - dimensions: - interval: biweekly - metric: per_capita - indicators: - y: "{definitions.table}#biweekly_deaths_per_million" - - dimensions: - interval: biweekly - metric: change - indicators: - y: "{definitions.table}#biweekly_pct_growth_deaths" + # Views will be filled out programmatically. + [] + ``` -The `dimensions` field specifies selectors, and the `views` field defines views for the selection. Since there are numerous possible configurations, `views` are usually generated programmatically. However, it's a good idea to create a few of them manually to start. +The `dimensions` field specifies selectors, and the `views` field defines views for the selection. Since there are numerous possible configurations, `views` are usually generated programmatically (using function `etl.multidim.generate_views_for_dimensions`). You can also combine manually defined views with generated ones. See the `etl.multidim` module for available helper functions or refer to examples from `etl/steps/export/multidim/`. Feel free to add or modify the helper functions as needed. -The export step loads the YAML file, adds `views` to the config, and then calls the function. +The export step loads the data dependencies and the config YAML file, adds `views` to the config, and then pushes the configuration to the database. -```python title="etl/steps/export/multidim/covid/latest/covid.py" +```python title="etl/steps/export/multidim/energy/latest/energy_prices.py" def run(dest_dir: str) -> None: - engine = get_engine() - - # Load YAML file - config = paths.load_mdim_config("covid.deaths.yaml") + # + # Load inputs. + # + # Load data on energy prices. + ds_grapher = paths.load_dataset("energy_prices") + + # Read table of prices in euros. + tb_annual = ds_grapher.read("energy_prices_annual") + tb_monthly = ds_grapher.read("energy_prices_monthly") + + # + # Process data. + # + # Load configuration from adjacent yaml file. + config = paths.load_mdim_config() + + # Create views. + config["views"] = multidim.generate_views_for_dimensions( + dimensions=config["dimensions"], + tables=[tb_annual, tb_monthly], + dimensions_order_in_slug=("frequency", "source", "unit"), + warn_on_missing_combinations=False, + additional_config={"chartTypes": ["LineChart"], "hasMapTab": True, "tab": "map"}, + ) + + # + # Save outputs. + # + multidim.upsert_multidim_data_page(slug="mdd-energy-prices", config=config, engine=get_engine()) - multidim.upsert_multidim_data_page("mdd-energy", config, engine) ``` To see the multi-dimensional indicator in Admin, run ```bash -etlr export://multidim/energy/latest/energy --export +etlr export://multidim/energy/latest/energy_prices --export ``` -and check out the preview at http://staging-site-my-branch/admin/grapher/mdd-name. +and check out the preview at: http://staging-site-my-branch/admin/grapher/mdd-energy-prices ## Exporting data to GitHub diff --git a/etl/helpers.py b/etl/helpers.py index ecc8df917b5..9b47f5ba353 100644 --- a/etl/helpers.py +++ b/etl/helpers.py @@ -594,7 +594,7 @@ def _get_attributes_from_step_name(step_name: str) -> Dict[str, str]: if channel_type.startswith(("walden", "snapshot")): channel = channel_type namespace, version, short_name = path.split("/") - elif channel_type.startswith(("data",)): + elif channel_type.startswith(("data", "export")): channel, namespace, version, short_name = path.split("/") else: raise WrongStepName diff --git a/etl/multidim.py b/etl/multidim.py index 0931cb0f946..f22c68cc87a 100644 --- a/etl/multidim.py +++ b/etl/multidim.py @@ -1,14 +1,20 @@ import json +from itertools import product import pandas as pd import yaml from sqlalchemy.engine import Engine +from structlog import get_logger from apps.chart_sync.admin_api import AdminAPI from etl.config import OWID_ENV from etl.db import read_sql +from etl.grapher_io import trim_long_variable_name from etl.helpers import map_indicator_path_to_id +# Initialize logger. +log = get_logger() + def upsert_multidim_data_page(slug: str, config: dict, engine: Engine) -> None: validate_multidim_config(config, engine) @@ -162,3 +168,103 @@ def fetch_variables_from_table(table: str, engine: Engine) -> pd.DataFrame: df_dims = pd.DataFrame(dims, index=df.index) return df.join(df_dims) + + +def generate_views_for_dimensions( + dimensions, tables, dimensions_order_in_slug=None, additional_config=None, warn_on_missing_combinations=True +): + """Generate individual views for all possible combinations of dimensions in a list of flattened tables. + + Parameters + ---------- + dimensions : List[Dict[str, Any]] + Dimensions, as given in the configuration of the multidim step, e.g. + [ + {'slug': 'frequency', 'name': 'Frequency', 'choices': [{'slug': 'annual','name': 'Annual'}, {'slug': 'monthly', 'name': 'Monthly'}]}, + {'slug': 'source', 'name': 'Energy source', 'choices': [{'slug': 'electricity', 'name': 'Electricity'}, {'slug': 'gas', 'name': 'Gas'}]}, + ... + ] + tables : List[Table] + Tables whose indicator views will be generated. + dimensions_order_in_slug : Tuple[str], optional + Dimension names, as they appear in "dimensions", and in the order in which they are spelled out in indicator names. For example, if indicator names are, e.g. annual_electricity_euros, then dimensions_order_in_slug would be ("frequency", "source", "unit"). + additional_config : _type_, optional + Additional config fields to add to each view, e.g. + {"chartTypes": ["LineChart"], "hasMapTab": True, "tab": "map"} + warn_on_missing_combinations : bool, optional + True to warn if any combination of dimensions is not found among the indicators in the given tables. + + Returns + ------- + results : List[Dict[str, Any]] + Views configuration, e.g. + [ + {'dimensions': {'frequency': 'annual', 'source': 'electricity', 'unit': 'euro'}, 'indicators': {'y': 'grapher/energy/2024-11-20/energy_prices/energy_prices_annual#annual_electricity_household_total_price_including_taxes_euro'}, + {'dimensions': {'frequency': 'annual', 'source': 'electricity', 'unit': 'pps'}, 'indicators': {'y': 'grapher/energy/2024-11-20/energy_prices/energy_prices_annual#annual_electricity_household_total_price_including_taxes_pps'}, + ... + ] + + """ + # Extract all choices for each dimension as (slug, choice_slug) pairs. + choices = {dim["slug"]: [choice["slug"] for choice in dim["choices"]] for dim in dimensions} + dimension_slugs_in_config = set(choices.keys()) + + # Sanity check for dimensions_order_in_slug. + if dimensions_order_in_slug: + dimension_slugs_in_order = set(dimensions_order_in_slug) + + # Check if any slug in the order is missing from the config. + missing_slugs = dimension_slugs_in_order - dimension_slugs_in_config + if missing_slugs: + raise ValueError( + f"The following dimensions are in 'dimensions_order_in_slug' but not in the config: {missing_slugs}" + ) + + # Check if any slug in the config is missing from the order. + extra_slugs = dimension_slugs_in_config - dimension_slugs_in_order + if extra_slugs: + log.warning( + f"The following dimensions are in the config but not in 'dimensions_order_in_slug': {extra_slugs}" + ) + + # Reorder choices to match the specified order. + choices = {dim: choices[dim] for dim in dimensions_order_in_slug if dim in choices} + + # Generate all combinations of the choices. + all_combinations = list(product(*choices.values())) + + # Create the views. + results = [] + for combination in all_combinations: + # Map dimension slugs to the chosen values. + dimension_mapping = {dim_slug: choice for dim_slug, choice in zip(choices.keys(), combination)} + slug_combination = "_".join(combination) + + # Find relevant tables for the current combination. + relevant_table = [] + for table in tables: + if slug_combination in table: + relevant_table.append(table) + + # Handle missing or multiple table matches. + if len(relevant_table) == 0: + if warn_on_missing_combinations: + log.warning(f"Combination {slug_combination} not found in tables") + continue + elif len(relevant_table) > 1: + log.warning(f"Combination {slug_combination} found in multiple tables: {relevant_table}") + + # Construct the indicator path. + indicator_path = f"{relevant_table[0].metadata.dataset.uri}/{relevant_table[0].metadata.short_name}#{trim_long_variable_name(slug_combination)}" + indicators = { + "y": indicator_path, + } + # Append the combination to results. + results.append({"dimensions": dimension_mapping, "indicators": indicators}) + + if additional_config: + # Include additional fields in all results. + for result in results: + result.update({"config": additional_config}) + + return results diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json new file mode 100644 index 00000000000..48aa385d8ee --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.countries.json @@ -0,0 +1,31 @@ +{ + "Austria": "Austria", + "Belgium": "Belgium", + "Bulgaria": "Bulgaria", + "Croatia": "Croatia", + "Czechia": "Czechia", + "Denmark": "Denmark", + "Estonia": "Estonia", + "Finland": "Finland", + "France": "France", + "Germany": "Germany", + "Greece": "Greece", + "Hungary": "Hungary", + "Ireland": "Ireland", + "Italy": "Italy", + "Latvia": "Latvia", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Netherlands": "Netherlands", + "North Macedonia": "North Macedonia", + "Norway": "Norway", + "Poland": "Poland", + "Portugal": "Portugal", + "Romania": "Romania", + "Serbia": "Serbia", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Spain": "Spain", + "Sweden": "Sweden", + "Switzerland": "Switzerland" +} diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml new file mode 100644 index 00000000000..98ad5c38e4e --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.meta.yml @@ -0,0 +1,27 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + processing_level: minor + +dataset: + update_period_days: 365 + +tables: + european_wholesale_electricity_prices_monthly: + variables: + price: + title: Electricity wholesale monthly price + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + description_short: |- + Monthly average day-ahead spot prices per [megawatt-hour](#dod:watt-hours) of electricity sold. + european_wholesale_electricity_prices_annual: + variables: + price: + title: Electricity wholesale annual price + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + description_short: |- + Annual average day-ahead spot prices per [megawatt-hour](#dod:watt-hours) of electricity sold. diff --git a/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py new file mode 100644 index 00000000000..e6a27829c94 --- /dev/null +++ b/etl/steps/data/garden/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -0,0 +1,57 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Select and rename columns. +COLUMNS = { + "country": "country", + "date": "date", + "price__eur_mwhe": "price", +} + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("european_wholesale_electricity_prices") + + # Read table from meadow dataset. + tb_monthly = ds_meadow.read("european_wholesale_electricity_prices") + + # + # Process data. + # + # Select and rename columns. + tb_monthly = tb_monthly[list(COLUMNS)].rename(columns=COLUMNS, errors="raise") + + # Harmonize country names. + tb_monthly = geo.harmonize_countries(df=tb_monthly, countries_file=paths.country_mapping_path) + + # Ember provides monthly data, so we can create a monthly table of wholesale electricity prices. + # But we also need to create an annual table of average wholesale electricity prices. + tb_annual = tb_monthly.copy() + tb_annual["year"] = tb_annual["date"].str[:4].astype("Int64") + # NOTE: We will include only complete years. This means that the latest year will not be included. But also, we will disregard country-years like Ireland 2022, which only has data for a few months, for some reason. + n_months = tb_annual.groupby(["country", "year"], observed=True, as_index=False)["date"].transform("count") + tb_annual = ( + tb_annual[n_months == 12].groupby(["country", "year"], observed=True, as_index=False).agg({"price": "mean"}) + ) + + # Improve table formats. + tb_monthly = tb_monthly.format(["country", "date"], short_name="european_wholesale_electricity_prices_monthly") + tb_annual = tb_annual.format(short_name="european_wholesale_electricity_prices_annual") + + # + # Save outputs. + # + # Create a new garden dataset. + ds_garden = create_dataset(dest_dir, tables=[tb_monthly, tb_annual], check_variables_metadata=True) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml b/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml new file mode 100644 index 00000000000..6a934bc8bd5 --- /dev/null +++ b/etl/steps/data/garden/energy/2024-11-20/energy_prices.meta.yml @@ -0,0 +1,11 @@ +definitions: + common: + processing_level: major + presentation: + topic_tags: + - Energy + +dataset: + title: European Energy Prices + update_period_days: 365 + diff --git a/etl/steps/data/garden/energy/2024-11-20/energy_prices.py b/etl/steps/data/garden/energy/2024-11-20/energy_prices.py new file mode 100644 index 00000000000..f40c3b4fb44 --- /dev/null +++ b/etl/steps/data/garden/energy/2024-11-20/energy_prices.py @@ -0,0 +1,61 @@ +"""Compilation of energy prices datasets. + +""" +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current data step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load data. + # + # Load Eurostat data on European gas and electricity prices. + ds_eurostat = paths.load_dataset("gas_and_electricity_prices") + tb_eurostat_euro = ds_eurostat.read("gas_and_electricity_price_components_euro_flat") + tb_eurostat_pps = ds_eurostat.read("gas_and_electricity_price_components_pps_flat") + + # Load Ember data on European wholesale electricity prices. + ds_ember = paths.load_dataset("european_wholesale_electricity_prices") + tb_ember_monthly = ds_ember.read("european_wholesale_electricity_prices_monthly") + tb_ember_annual = ds_ember.read("european_wholesale_electricity_prices_annual") + + # + # Process data. + # + # Rename columns in all tables to have consistent dimensions. + tb_eurostat_euro = tb_eurostat_euro.rename( + columns={ + column: f"annual_{column}" for column in tb_eurostat_euro.columns if column not in ["country", "year"] + }, + errors="raise", + ) + tb_eurostat_pps = tb_eurostat_pps.rename( + columns={column: f"annual_{column}" for column in tb_eurostat_pps.columns if column not in ["country", "year"]}, + errors="raise", + ) + tb_ember_monthly = tb_ember_monthly.rename( + columns={"price": "monthly_electricity_all_wholesale_euro"}, errors="raise" + ) + tb_ember_annual = tb_ember_annual.rename(columns={"price": "annual_electricity_all_wholesale_euro"}, errors="raise") + + # Create a combined annual table. + tb_annual = pr.multi_merge( + tables=[tb_eurostat_euro, tb_eurostat_pps, tb_ember_annual], on=["country", "year"], how="outer" + ) + tb_annual = tb_annual.format(short_name="energy_prices_annual") + + # Create a combined monthly table. + # For now, only Ember has monthly data. + tb_monthly = tb_ember_monthly.copy() + tb_monthly = tb_monthly.format(keys=["country", "date"], short_name="energy_prices_monthly") + + # + # Save outputs. + # + # Create a new dataset with the same metadata as meadow + ds_garden = create_dataset(dest_dir=dest_dir, tables=[tb_annual, tb_monthly], check_variables_metadata=True) + ds_garden.save() diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json new file mode 100644 index 00000000000..f11ceb2730a --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.countries.json @@ -0,0 +1,44 @@ +{ + "BE": "Belgium", + "BG": "Bulgaria", + "CZ": "Czechia", + "DK": "Denmark", + "DE": "Germany", + "EE": "Estonia", + "IE": "Ireland", + "EL": "Greece", + "ES": "Spain", + "FR": "France", + "HR": "Croatia", + "IT": "Italy", + "CY": "Cyprus", + "LV": "Latvia", + "LT": "Lithuania", + "LU": "Luxembourg", + "HU": "Hungary", + "MT": "Malta", + "NL": "Netherlands", + "AT": "Austria", + "PL": "Poland", + "PT": "Portugal", + "RO": "Romania", + "SI": "Slovenia", + "SK": "Slovakia", + "FI": "Finland", + "SE": "Sweden", + "IS": "Iceland", + "LI": "Liechtenstein", + "NO": "Norway", + "BA": "Bosnia and Herzegovina", + "ME": "Montenegro", + "MD": "Moldova", + "GE": "Georgia", + "MK": "North Macedonia", + "AL": "Albania", + "RS": "Serbia", + "TR": "Turkey", + "XK": "Kosovo", + "EU27_2020": "European Union (27)", + "UA": "Ukraine", + "UK": "United Kingdom" +} diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json new file mode 100644 index 00000000000..7745db9ff44 --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.excluded_countries.json @@ -0,0 +1 @@ +["EA"] diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml new file mode 100644 index 00000000000..5b27ffbfd55 --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.meta.yml @@ -0,0 +1,1334 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + processing_level: major + + # Definitions of consumer type: + description_key_household_consumer: |- + A household consumer of gas or electricity is a residential user who consumes energy for domestic purposes like heating, cooking, lighting, and appliances. + description_key_non_household_consumer: |- + A non-household consumer of gas or electricity is a business or organization using energy for purposes like industry, services, offices, or agriculture. + # Definitions of consumer bands: + description_key_all_consumer_bands: |- + Prices are a weighted average across all consumption bands, from low to high energy consumers, based on the relative consumption shares reported by each country. + # Definitions of price components: + # Eurostat's definitions can be found: + # * For gas: https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_202_sims.htm + # * For electricity: https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_204_sims.htm + # * Gas - Energy and supply: commodity price for natural gas paid by the supplier or the price of natural gas at the point of entry into the transmission system, including, if applicable, the following end-user costs: storage costs plus costs relating to the sale of natural gas to final customers. + # * Electricity - Energy and supply: generation, aggregation, balancing energy, supplied energy costs, customer services, after-sales management and other supply costs. + # * Gas/electricity - Network cost: transmission and distribution tariffs, transmission and distribution losses, network costs, after-sale service costs, system service costs, and meter rental and metering costs. + # * Gas/electricity - Value added taxes (VAT): as defined in Council Directive 2006/112/EC. + # * Gas/electricity - Renewable taxes: taxes, fees, levies or charges relating to the promotion of renewable energy sources, energy efficiency and CHP generation. + # * Gas - Capacity taxes: taxes, fees, levies or charges relating to strategic stockpiles, capacity payments and energy security; taxes on natural gas distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Electricity - Capacity taxes: Taxes, fees, levies or charges relating to capacity payments, energy security and generation adequacy; taxes on coal industry restructuring; taxes on electricity distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Gas/electricity - Environmental taxes: taxes, fees, levies or charges relating to air quality and for other environmental purposes; taxes on emissions of CO2 or other greenhouse gases. This component includes the excise duties. + # * Gas/electricity - All other taxes: taxes, fees, levies or charges not covered by any of the previous four categories: support for district heating; local or regional fiscal charges; island compensation; concession fees relating to licences and fees for the occupation of land and public or private property by networks or other devices. + # * Electricity - Nuclear taxes: taxes, fees, levies or charges relating to the nuclear sector, including nuclear decommissioning, inspections and fees for nuclear installations. + description_key_gas_component_energy_and_supply: |- + "Energy and supply" is the cost of natural gas, including the commodity price paid by the supplier or the price at the point of entry into the transmission system. It also includes, where applicable, end-user costs such as storage and expenses related to the sale of gas to final customers. + description_key_gas_component_network_cost: &description_key_gas_component_network_cost |- + "Network costs" include the tariffs for transmission and distribution, costs from transmission and distribution losses, and system-related expenses such as after-sale services, system services, meter rental, and metering. + description_key_gas_component_taxes_fees_levies_and_charges: &description_key_gas_component_taxes_fees_levies_and_charges |- + "Taxes, fees, levies, and charges" include value-added tax (VAT), renewable taxes, capacity taxes, environmental taxes, and all other taxes, fees, levies, or charges. + description_key_gas_component_vat: &description_key_gas_component_vat |- + "Value-added tax (VAT)" is a consumption tax applied to the final price of gas or electricity, as defined in Council Directive 2006/112/EC. + description_key_gas_component_renewable_taxes: &description_key_gas_component_renewable_taxes |- + "Renewable taxes" are taxes, fees, levies, or charges that promote renewable energy sources, energy efficiency, and combined heat and power (CHP) generation. + description_key_gas_component_capacity_taxes: |- + "Capacity taxes" are charges related to maintaining energy security and the gas distribution system. They include taxes or fees for strategic gas stockpiles, payments to ensure sufficient supply capacity, costs for supporting energy regulatory authorities or system operators, and levies to cover past infrastructure investments (stranded costs). + description_key_gas_component_environmental_taxes: &description_key_gas_component_environmental_taxes |- + "Environmental taxes" are charges aimed at addressing environmental concerns, such as improving air quality and reducing greenhouse gas emissions (e.g., CO₂ taxes). This category also includes excise duties—specific taxes applied to goods or activities like fuels and energy use, which contribute to environmental impacts. + description_key_gas_component_other_taxes: &description_key_gas_component_other_taxes |- + "All other taxes" include fees, levies, or charges not covered by other tax components. They may relate to district heating support, local or regional fiscal charges, island compensation, concession fees for licenses, or fees for occupying land or property with networks or other devices. + description_key_gas_component_total_price_including_taxes: &description_key_gas_component_total_price_including_taxes |- + "Total price including taxes" is the sum of all price components, including energy and supply, network costs, taxes, fees, levies and charges. + description_key_electricity_component_energy_and_supply: |- + "Energy and supply" is the cost of electricity generation, aggregation, balancing, and other supply-related activities, including end-user costs such as customer services and after-sales management. + description_key_electricity_component_network_cost: *description_key_gas_component_network_cost + description_key_electricity_component_vat: *description_key_gas_component_vat + description_key_electricity_component_renewable_taxes: *description_key_gas_component_renewable_taxes + description_key_electricity_component_capacity_taxes: |- + "Capacity taxes" are charges aimed at ensuring energy security and sufficient electricity supply. They include fees for maintaining generation capacity, supporting energy regulatory authorities or system operators, restructuring the coal industry, and improving electricity distribution. They may also cover costs from past infrastructure investments (stranded costs). + description_key_electricity_component_environmental_taxes: *description_key_gas_component_environmental_taxes + description_key_electricity_component_other_taxes: *description_key_gas_component_other_taxes + description_key_electricity_component_nuclear_taxes: |- + "Nuclear taxes" are charges related to the nuclear sector, including decommissioning, inspections, and fees for nuclear installations. + description_key_electricity_component_total_price_including_taxes: *description_key_gas_component_total_price_including_taxes + description_key_electricity_component_taxes_fees_levies_and_charges: *description_key_gas_component_taxes_fees_levies_and_charges + # Definitions of price levels: + # Original Eurostat definitions of price levels: + # Level 1 prices ("X_TAX"): Prices excluding taxes and levies. + # Level 2 prices ("X_VAT"): Prices excluding VAT and other recoverable taxes and levies. + # Level 3 prices ("I_TAX"): Prices including all taxes and levies. + description_key_price_excluding_taxes: |- + Prices represent the base cost of energy, excluding all taxes, levies, and VAT. These prices reflect the pure costs of energy and supply, network, and other market-related services. + description_key_price_excluding_vat: |- + Prices include the base cost of energy and non-recoverable taxes and levies. They exclude VAT and other recoverable taxes. + description_key_price_including_all_taxes: |- + Prices represent the total price paid by end consumers, including all taxes, levies, and VAT. + + +dataset: + update_period_days: 365 + + +tables: + gas_and_electricity_prices: + variables: + price_euro: + title: Price + unit: euro + short_unit: "€" + description_short: Energy price in euro. + price_pps: + title: Price (purchasing power) + unit: purchasing power standard + short_unit: PPS + description_short: Energy price in purchasing power standard. + gas_and_electricity_price_components_euro_flat: + title: Gas and electricity price components in Europe + common: + description_short: Price components are given in euros per [megawatt-hour](#dod:watt-hours). They are not adjusted for inflation or differences in living costs between countries. + unit: 'current euros per megawatt-hour' + short_unit: "€/MWh" + variables: + electricity_household_capacity_taxes_euro: + title: Electricity price component (euros) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component for household consumers - Capacity taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_capacity_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component for non-household consumers - Capacity taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_capacity_taxes_allowances_euro: + title: Electricity price component (euros) for household consumers - Capacity taxes allowances + display: + name: Capacity taxes allowances + presentation: + title_public: Electricity price component for household consumers - Capacity taxes allowances + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_capacity_taxes_allowances}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_energy_and_supply_euro: + title: Electricity price component (euros) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component for household consumers - Energy and supply + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_energy_and_supply_euro: + title: Electricity price component (euros) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component for non-household consumers - Energy and supply + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_euro: + title: Electricity price component (euros) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component for household consumers - Environmental taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_environmental_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component for non-household consumers - Environmental taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Environmental taxes allowance + display: + name: Environmental taxes allowance + presentation: + title_public: Electricity price component for household consumers - Environmental taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_environmental_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_network_costs_euro: + title: Electricity price component (euros) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component for household consumers - Network costs + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_network_costs_euro: + title: Electricity price component (euros) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component for non-household consumers - Network costs + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_euro: + title: Electricity price component (euros) for household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component for household consumers - Nuclear taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_nuclear_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component for non-household consumers - Nuclear taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Nuclear taxes allowance + display: + name: Nuclear taxes allowance + presentation: + title_public: Electricity price component for household consumers - Nuclear taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_nuclear_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_euro: + title: Electricity price component (euros) for household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component for household consumers - Other + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_other_euro: + title: Electricity price component (euros) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component for non-household consumers - Other + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_allowance_euro: + title: Electricity price component (euros) for household consumers - Other allowance + display: + name: Other allowance + presentation: + title_public: Electricity price component for household consumers - Other allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_other_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_euro: + title: Electricity price component (euros) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component for household consumers - Renewable taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_renewable_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component for non-household consumers - Renewable taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_allowance_euro: + title: Electricity price component (euros) for household consumers - Renewable taxes allowance + display: + name: Renewable taxes allowance + presentation: + title_public: Electricity price component for household consumers - Renewable taxes allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_renewable_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_euro: + title: Electricity price component (euros) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component for household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_taxes_fees_levies_and_charges_euro: + title: Electricity price component (euros) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component for non-household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_value_added_tax_vat_euro: + title: Electricity price component (euros) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component for household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_value_added_tax_vat_euro: + title: Electricity price component (euros) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component for non-household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_capacity_taxes_euro: + title: Gas price component (euros) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component for household consumers - Capacity taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_energy_and_supply_euro: + title: Gas price component (euros) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component for household consumers - Energy and supply + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_environmental_taxes_euro: + title: Gas price component (euros) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component for household consumers - Environmental taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_network_costs_euro: + title: Gas price component (euros) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component for household consumers - Network costs + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_other_euro: + title: Gas price component (euros) for household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component for household consumers - Other + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_renewable_taxes_euro: + title: Gas price component (euros) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component for household consumers - Renewable taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_taxes_fees_levies_and_charges_euro: + title: Gas price component (euros) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component for household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_value_added_tax_vat_euro: + title: Gas price component (euros) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component for household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_capacity_taxes_euro: + title: Gas price component (euros) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component for non-household consumers - Capacity taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_energy_and_supply_euro: + title: Gas price component (euros) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component for non-household consumers - Energy and supply + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_environmental_taxes_euro: + title: Gas price component (euros) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component for non-household consumers - Environmental taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_network_costs_euro: + title: Gas price component (euros) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component for non-household consumers - Network costs + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_other_euro: + title: Gas price component (euros) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component for non-household consumers - Other + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_renewable_taxes_euro: + title: Gas price component (euros) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component for non-household consumers - Renewable taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_taxes_fees_levies_and_charges_euro: + title: Gas price component (euros) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component for non-household consumers - Taxes, fees, levies, and charges + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_value_added_tax_vat_euro: + title: Gas price component (euros) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component for non-household consumers - Value added tax (VAT) + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_allowance_euro: + title: Electricity price component (euros) for household consumers - Taxes, fees, levies, and charges allowance + display: + name: Taxes, fees, levies, and charges allowance + presentation: + title_public: Electricity price component for household consumers - Taxes, fees, levies, and charges allowance + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_total_price_including_taxes_euro: + title: Electricity price component (euros) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component for household consumers - Total price including taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_total_price_including_taxes_euro: + title: Electricity price component (euros) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component for non-household consumers - Total price including taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_total_price_including_taxes_euro: + title: Gas price component (euros) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component for household consumers - Total price including taxes + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_total_price_including_taxes_euro: + title: Gas price component (euros) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component for non-household consumers - Total price including taxes + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_and_electricity_price_components_pps_flat: + title: Gas and electricity price components in Europe (PPS) + common: + description_short: Price components are given in purchasing power standard (PPS) per [megawatt-hour](#dod:watt-hours). + unit: "purchasing power standard per megawatt-hour" + short_unit: "PPS/MWh" + variables: + electricity_household_capacity_taxes_pps: + title: Electricity price component (PPS) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_capacity_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_capacity_taxes_allowances_pps: + title: Electricity price component (PPS) for household consumers - Capacity taxes allowances + display: + name: Capacity taxes allowances + presentation: + title_public: Electricity price component (PPS) for household consumers - Capacity taxes allowances + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_capacity_taxes_allowances}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_energy_and_supply_pps: + title: Electricity price component (PPS) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component (PPS) for household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_energy_and_supply_pps: + title: Electricity price component (PPS) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_pps: + title: Electricity price component (PPS) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_environmental_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_environmental_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Environmental taxes allowance + display: + name: Environmental taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Environmental taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_environmental_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_network_costs_pps: + title: Electricity price component (PPS) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component (PPS) for household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_network_costs_pps: + title: Electricity price component (PPS) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_pps: + title: Electricity price component (PPS) for household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Nuclear taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_nuclear_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Nuclear taxes + display: + name: Nuclear taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Nuclear taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_nuclear_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_nuclear_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Nuclear taxes allowance + display: + name: Nuclear taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Nuclear taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_nuclear_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_pps: + title: Electricity price component (PPS) for household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component (PPS) for household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_other_pps: + title: Electricity price component (PPS) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_other_allowance_pps: + title: Electricity price component (PPS) for household consumers - Other allowance + display: + name: Other allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Other allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_other_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_pps: + title: Electricity price component (PPS) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_renewable_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_renewable_taxes_allowance_pps: + title: Electricity price component (PPS) for household consumers - Renewable taxes allowance + display: + name: Renewable taxes allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Renewable taxes allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_renewable_taxes_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_pps: + title: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_taxes_fees_levies_and_charges_pps: + title: Electricity price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_value_added_tax_vat_pps: + title: Electricity price component (PPS) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component (PPS) for household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_value_added_tax_vat_pps: + title: Electricity price component (PPS) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_capacity_taxes_pps: + title: Gas price component (PPS) for household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_energy_and_supply_pps: + title: Gas price component (PPS) for household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component (PPS) for household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_environmental_taxes_pps: + title: Gas price component (PPS) for household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_network_costs_pps: + title: Gas price component (PPS) for household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component (PPS) for household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_other_pps: + title: Gas price component (PPS) for household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component (PPS) for household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_renewable_taxes_pps: + title: Gas price component (PPS) for household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_taxes_fees_levies_and_charges_pps: + title: Gas price component (PPS) for household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component (PPS) for household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_value_added_tax_vat_pps: + title: Gas price component (PPS) for household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component (PPS) for household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_capacity_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Capacity taxes + display: + name: Capacity taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Capacity taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_capacity_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_energy_and_supply_pps: + title: Gas price component (PPS) for non-household consumers - Energy and supply + display: + name: Energy and supply + presentation: + title_public: Gas price component (PPS) for non-household consumers - Energy and supply + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_energy_and_supply}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_environmental_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Environmental taxes + display: + name: Environmental taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Environmental taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_environmental_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_network_costs_pps: + title: Gas price component (PPS) for non-household consumers - Network costs + display: + name: Network costs + presentation: + title_public: Gas price component (PPS) for non-household consumers - Network costs + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_network_cost}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_other_pps: + title: Gas price component (PPS) for non-household consumers - Other + display: + name: Other + presentation: + title_public: Gas price component (PPS) for non-household consumers - Other + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_other_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_renewable_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Renewable taxes + display: + name: Renewable taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Renewable taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_renewable_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_taxes_fees_levies_and_charges_pps: + title: Gas price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + display: + name: Taxes, fees, levies, and charges + presentation: + title_public: Gas price component (PPS) for non-household consumers - Taxes, fees, levies, and charges + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_taxes_fees_levies_and_charges}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_value_added_tax_vat_pps: + title: Gas price component (PPS) for non-household consumers - Value added tax (VAT) + display: + name: Value added tax (VAT) + presentation: + title_public: Gas price component (PPS) for non-household consumers - Value added tax (VAT) + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_vat}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_taxes_fees_levies_and_charges_allowance_pps: + title: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges allowance + display: + name: Taxes, fees, levies, and charges allowance + presentation: + title_public: Electricity price component (PPS) for household consumers - Taxes, fees, levies, and charges allowance + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + # - "{definitions.description_key_electricity_component_taxes_fees_levies_and_charges_allowance}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_household_total_price_including_taxes_pps: + title: Electricity price component (PPS) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component (PPS) for household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + electricity_non_household_total_price_including_taxes_pps: + title: Electricity price component (PPS) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Electricity price component (PPS) for non-household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_electricity_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_household_total_price_including_taxes_pps: + title: Gas price component (PPS) for household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component (PPS) for household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_non_household_total_price_including_taxes_pps: + title: Gas price component (PPS) for non-household consumers - Total price including taxes + display: + name: Total price including taxes + presentation: + title_public: Gas price component (PPS) for non-household consumers - Total price including taxes + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_gas_component_total_price_including_taxes}" + - "{definitions.description_key_all_consumer_bands}" + + gas_and_electricity_prices_euro_flat: + title: Gas and electricity prices in Europe + common: + description_short: Prices are given in euros per [megawatt-hour](#dod:watt-hours). They are not adjusted for inflation or differences in living costs between countries. + unit: "current euros per megawatt-hour" + short_unit: "€/MWh" + variables: + electricity_household_all_taxes_and_levies_included_euro: + title: Electricity price (euros) for household consumers - All taxes and levies included + presentation: + title_public: Electricity price for household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_household_excluding_taxes_and_levies_euro: + title: Electricity price (euros) for household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Electricity price (euros) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + electricity_non_household_all_taxes_and_levies_included_euro: + title: Electricity price (euros) for non-household consumers - All taxes and levies included + presentation: + title_public: Electricity price for non-household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_non_household_excluding_taxes_and_levies_euro: + title: Electricity price (euros) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Electricity price (euros) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_household_all_taxes_and_levies_included_euro: + title: Gas price (euros) for household consumers - All taxes and levies included + presentation: + title_public: Gas price for household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_household_excluding_taxes_and_levies_euro: + title: Gas price (euros) for household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Gas price (euros) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_non_household_all_taxes_and_levies_included_euro: + title: Gas price (euros) for non-household consumers - All taxes and levies included + presentation: + title_public: Gas price for non-household consumers - All taxes and levies included + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_non_household_excluding_taxes_and_levies_euro: + title: Gas price (euros) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_euro: + title: Gas price (euros) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding VAT and other recoverable taxes and levies + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_and_electricity_prices_pps_flat: + title: Gas and electricity prices in Europe (PPS) + common: + description_short: Prices are given in purchasing power standard (PPS) per [megawatt-hour](#dod:watt-hours). + unit: "purchasing power standard per megawatt-hour" + short_unit: "PPS/MWh" + variables: + electricity_household_all_taxes_and_levies_included_pps: + title: Electricity price (PPS) for household consumers - All taxes and levies included + presentation: + title_public: Electricity price for household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_household_excluding_taxes_and_levies_pps: + title: Electricity price (PPS) for household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Electricity price (PPS) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + electricity_non_household_all_taxes_and_levies_included_pps: + title: Electricity price (PPS) for non-household consumers - All taxes and levies included + presentation: + title_public: Electricity price for non-household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + electricity_non_household_excluding_taxes_and_levies_pps: + title: Electricity price (PPS) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + electricity_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Electricity price (PPS) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Electricity price for non-household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_household_all_taxes_and_levies_included_pps: + title: Gas price (PPS) for household consumers - All taxes and levies included + presentation: + title_public: Gas price for household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_household_excluding_taxes_and_levies_pps: + title: Gas price (PPS) for household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Gas price (PPS) for household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" + gas_non_household_all_taxes_and_levies_included_pps: + title: Gas price (PPS) for non-household consumers - All taxes and levies included + presentation: + title_public: Gas price for non-household consumers - All taxes and levies included + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_including_all_taxes}" + gas_non_household_excluding_taxes_and_levies_pps: + title: Gas price (PPS) for non-household consumers - Excluding taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_taxes}" + gas_non_household_excluding_vat_and_other_recoverable_taxes_and_levies_pps: + title: Gas price (PPS) for non-household consumers - Excluding VAT and other recoverable taxes and levies + presentation: + title_public: Gas price for non-household consumers - Excluding VAT and other recoverable taxes and levies + title_variant: PPS + description_key: + - "{definitions.description_key_non_household_consumer}" + - "{definitions.description_key_all_consumer_bands}" + - "{definitions.description_key_price_excluding_vat}" diff --git a/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..afc793eaabb --- /dev/null +++ b/etl/steps/data/garden/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,895 @@ +"""Load a meadow dataset and create a garden dataset.""" +from typing import Dict + +import owid.catalog.processing as pr +import pandas as pd +import plotly.express as px +from owid.catalog import Table +from owid.datautils.dataframes import map_series + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Dataset codes to select, and their corresponding names. +DATASET_CODES_AND_NAMES = { + #################################################################################################################### + # Gas and electricity prices. + # NOTE: Prices are given per semester. + "nrg_pc_202": "Gas prices for household consumers", # bi-annual data (from 2007) + "nrg_pc_203": "Gas prices for non-household consumers", # bi-annual data (from 2007) + "nrg_pc_204": "Electricity prices for household consumers", # bi-annual data (from 2007) + "nrg_pc_205": "Electricity prices for non-household consumers", # bi-annual data (from 2007) + #################################################################################################################### + # Gas and electricity prices components. + "nrg_pc_202_c": "Gas prices components for household consumers", # annual data (from 2007) + "nrg_pc_203_c": "Gas prices components for non-household consumers", # annual data (from 2007) + "nrg_pc_204_c": "Electricity prices components for household consumers", # annual data (from 2007) + "nrg_pc_205_c": "Electricity prices components for non-household consumers", # annual data (from 2007) + #################################################################################################################### + # Historical data. + # NOTE: For now I think we will have to ignore historical data. + # I doesn't have a band for total price. Instead, it has different consumption bands (defined by "consom"). + # This field is a bit problematic. + # The same value, e.g. "4141050" has different definitions for electricity ("Households - Da (annual consumption: 600 kWh)") and for gas ("Households - D1 (annual consumption: 8.37 GJ)"). + # The fact that the same value is used for different things is inconvenient, but not the main problem. + # The main problem is that we would need to figure out how to properly aggregate these values to get totals (meanwhile current data comes with totals). + # Additionally, historical data is disaggregated in "domestic" and "industrial", whereas current data is split in "households" and "non-households". + # "consom": {} + # "nrg_pc_202_h": "Gas prices for domestic consumers", # bi-annual data (until 2007) + # "nrg_pc_203_h": "Gas prices for industrial consumers", # bi-annual data (until 2007) + # "nrg_pc_204_h": "Electricity prices for domestic consumers", # bi-annual data (until 2007) + # "nrg_pc_205_h": "Electricity prices for industrial consumers", # bi-annual data (until 2007) + # "nrg_pc_206_h": "Electricity marker prices", # bi-annual data (until 2007) + #################################################################################################################### + # Share for transmission and distribution in the network cost for gas and electricity. + # NOTE: Decide if we could use the following. + # "nrg_pc_206": "Share for transmission and distribution in the network cost for gas and electricity", # annual data (from 2007) + #################################################################################################################### + # The following are consumption volumes of electricity by consumption bands. + # It doesn't give the relative consumption of each semester. If I understand correctly, it gives the percentage consumption of each band in the total consumption of the year. + # "nrg_pc_202_v": "Gas consumption volumes for households", # annual data (from 2007) + # "nrg_pc_203_v": "Gas consumption volumes for non-households", # annual data (from 2007) + # "nrg_pc_204_v": "Electricity consumption volumes for households", # annual data (from 2007) + # "nrg_pc_205_v": "Electricity consumption volumes for non-households", # annual data (from 2007) +} + +# Columns to keep and how to rename them. +COLUMNS = { + "nrg_cons": "consumption_band", + "unit": "energy_unit", + "tax": "price_level", + "currency": "currency", + "geo": "country", + "time": "year", + "dataset_code": "dataset_code", + "nrg_prc": "price_component", + "value": "value", +} + +# Mappings of indexes. +# The definitions are copied from (replace [DATASET_CODE] with the dataset code): +# https://ec.europa.eu/eurostat/databrowser/view/[DATASET_CODE]/default/table?lang=en&category=nrg.nrg_price.nrg_pc +INDEXES_MAPPING = { + # Currencies. + "currency": { + "EUR": "Euro", + # Purchasing Power Standard + "PPS": "PPS", + "NAC": "National currency", + "NAT": "National (former) currency", + }, + # Flags (found right next to the value, as a string). + # NOTE: Flag definitions are right below the data table in that page. + "flag": { + "e": "estimated", + "c": "confidential", + "d": "definition differs", + "b": "break in time series", + "p": "provisional", + "u": "low reliability", + "cd": "confidential, definition differs", + # NOTE: I couldn't find the meaning of the following flag. + # It happens for "Electricity prices for non-household consumers" for Cyprus in 2024 (for MWH_GE150000), and all values are zero. + "n": "unknown flag", + }, + # Price levels. + "price_level": { + # All taxes and levies included + "I_TAX": "All taxes and levies included", + # Excluding VAT and other recoverable taxes and levies + # NOTE: This value gives a baseline price for electricity before any additional costs imposed by taxes or fees are added. It represents the net price of electricity. + "X_TAX": "Excluding taxes and levies", + # Excluding value-added tax (VAT) and other recoverable taxes and levies + "X_VAT": "Excluding VAT and other recoverable taxes and levies", + }, + # Consumption bands. + # NOTE: This is only relevant for non-historical data. + "consumption_band": { + # Consumption bands for "Gas prices for household consumers" and "Gas price components for household consumers": + # Consumption of GJ - all bands + "TOT_GJ": "All bands", + # Consumption less than 20 GJ - band D1 + "GJ_LT20": "<20GJ", + # Consumption from 20 GJ to 199 GJ - band D2 + "GJ20-199": "20-199GJ", + # Consumption 200 GJ or over - band D3 + "GJ_GE200": ">=200GJ", + ################################################################################################################ + # Consumption bands for "Gas prices components for non-household consumers" and "Gas prices components for non-household consumers": + # 'TOT_GJ': "All bands", # Already defined above. + # Consumption less than 1 000 GJ - band I1 + "GJ_LT1000": "<1000GJ", + # Consumption from 1 000 GJ to 9 999 GJ -band I2 + "GJ1000-9999": "1000-9999GJ", + # Consumption from 10 000 GJ to 99 999 GJ - band I3 + "GJ10000-99999": "10000-99999GJ", + # Consumption from 100 000 GJ to 999 999 GJ - band I4 + "GJ100000-999999": "100000-999999GJ", + # Consumption from 1 000 000 GJ to 3 999 999 GJ - band I5 + "GJ1000000-3999999": "1000000-3999999GJ", + # Consumption 4 000 000 GJ or over - band I6 + "GJ_GE4000000": ">=4000000GJ", + ################################################################################################################ + # Consumption bands for "Electricity prices for household consumers" and "Electricity prices components for household consumers": + # Consumption of kWh - all bands + "TOT_KWH": "All bands", + # Consumption less than 1 000 kWh - band DA + "KWH_LT1000": "<1000kWh", + # Consumption from 1 000 kWh to 2 499 kWh - band DB + "KWH1000-2499": "1000-2499kWh", + # Consumption from 2 500 kWh to 4 999 kWh - band DC + "KWH2500-4999": "2500-4999kWh", + # Consumption from 5 000 kWh to 14 999 kWh - band DD + "KWH5000-14999": "5000-14999kWh", + # Consumption for 15 000 kWh or over - band DE + "KWH_GE15000": ">=15000kWh", + # NOTE: In the electricity components dataset, there is an additional band, which contains *LE* but in the metadata it seems to correspond to greater or equal, band DE, so it must be a typo in the band name. + # Consumption 15 000 kWh or over - band DE + "KWH_LE15000": ">=15000kWh", + ################################################################################################################ + # Consumption bands for "Electricity prices components for non-household consumers" and "Electricity prices components for non-household consumers": + # Consumption of kWh - all bands + # "TOT_KWH": "All bands", # Already defined above. + # Consumption less than 20 MWh - band IA + "MWH_LT20": "<20MWh", + # Consumption from 20 MWh to 499 MWh - band IB + "MWH20-499": "20-499MWh", + # Consumption from 500 MWh to 1 999 MWh - band IC + "MWH500-1999": "500-1999MWh", + # Consumption from 2 000 MWh to 19 999 MWh - band ID + "MWH2000-19999": "2000-19999MWh", + # Consumption from 20 000 MWh to 69 999 MWh - band IE + "MWH20000-69999": "20000-69999MWh", + # Consumption from 70 000 MWh to 149 999 MWh - band IF + "MWH70000-149999": "70000-149999MWh", + # Consumption 150 000 MWh or over - band IG + "MWH_GE150000": ">=150000MWh", + # NOTE: In the electricity components dataset, there is an additional band: + # Consumption 149 999 MWh or less - bandS IA-IF + "MWH_LE149999": "<=149999MWh", + #################################################################################################################### + }, + # Energy price components. + "price_component": { + # Gas prices components for household and non-household consumers + # Energy and supply + "NRG_SUP": "Energy and supply", + # Network costs + "NETC": "Network costs", + # Taxes, fees, levies and charges + "TAX_FEE_LEV_CHRG": "Taxes, fees, levies, and charges", + # Value added tax (VAT) + "VAT": "Value added tax (VAT)", + # Renewable taxes + "TAX_RNW": "Renewable taxes", + # Capacity taxes + "TAX_CAP": "Capacity taxes", + # Environmental taxes + "TAX_ENV": "Environmental taxes", + # Renewable taxes allowance + "TAX_RNW_ALLOW": "Renewable taxes allowance", + # Capacity taxes allowances + "TAX_CAP_ALLOW": "Capacity taxes allowances", + # Environmental taxes allowance + "TAX_ENV_ALLOW": "Environmental taxes allowance", + # Other allowance + "ALLOW_OTH": "Other allowance", + # Other + "OTH": "Other", + # Electricity prices components for household and non-household consumers + # All the above, plus the additional: + # Nuclear taxes + "TAX_NUC": "Nuclear taxes", + # Nuclear taxes allowance + "TAX_NUC_ALLOW": "Nuclear taxes allowance", + # Taxes, fees, levies and charges allowance + "TAX_FEE_LEV_CHRG_ALLOW": "Taxes, fees, levies, and charges allowance", + # From the metadata page (https://ec.europa.eu/eurostat/cache/metadata/en/nrg_pc_204_sims.htm), these are the components: + # * Energy and supply: generation, aggregation, balancing energy, supplied energy costs, customer services, after-sales management and other supply costs. + # * Network cost: transmission and distribution tariffs, transmission and distribution losses, network costs, after-sale service costs, system service costs, and meter rental and metering costs. + # * Value added taxes (VAT): as defined in Council Directive 2006/112/EC. + # * Renewable taxes: taxes, fees, levies or charges relating to the promotion of renewable energy sources, energy efficiency and CHP generation. + # * Capacity taxes: Taxes, fees, levies or charges relating to capacity payments, energy security and generation adequacy; taxes on coal industry restructuring; taxes on electricity distribution; stranded costs and levies on financing energy regulatory authorities or market and system operators. + # * Environmental taxes: taxes, fees, levies or charges relating to air quality and for other environmental purposes; taxes on emissions of CO2 or other greenhouse gases. This component includes the excise duties. + # * Nuclear taxes: taxes, fees, levies or charges relating to the nuclear sector, including nuclear decommissioning, inspections and fees for nuclear installations. + # * All other taxes: taxes, fees, levies or charges not covered by any of the previous five categories: support for district heating; local or regional fiscal charges; island compensation; concession fees relating to licences and fees for the occupation of land and public or private property by networks or other devices. + }, + # Energy units. + "energy_unit": { + # Gigajoule (gross calorific value - GCV) + "GJ_GCV": "GJ", + # Kilowatt-hour + "KWH": "kWh", + # The following is used in consumption volumes datasets. + # "PC": "Percentage", + }, +} + +# Dataset codes for prices and components. +DATASET_CODES_PRICES = ["nrg_pc_202", "nrg_pc_203", "nrg_pc_204", "nrg_pc_205"] +DATASET_CODES_COMPONENTS = ["nrg_pc_202_c", "nrg_pc_203_c", "nrg_pc_204_c", "nrg_pc_205_c"] +DATASET_CODE_TO_ENERGY_SOURCE = { + "nrg_pc_202": "Gas", + "nrg_pc_203": "Gas", + "nrg_pc_204": "Electricity", + "nrg_pc_205": "Electricity", + "nrg_pc_202_c": "Gas", + "nrg_pc_203_c": "Gas", + "nrg_pc_204_c": "Electricity", + "nrg_pc_205_c": "Electricity", +} +DATASET_CODE_TO_CONSUMER_TYPE_MAPPING = { + "nrg_pc_202": "Household", + "nrg_pc_203": "Non-household", + "nrg_pc_204": "Household", + "nrg_pc_205": "Non-household", + "nrg_pc_202_c": "Household", + "nrg_pc_203_c": "Non-household", + "nrg_pc_204_c": "Household", + "nrg_pc_205_c": "Non-household", +} + + +# The following components need to be present in the prices components datasets of a country-year-dataset-currency, otherwise its data will not be included. +MANDATORY_PRICE_COMPONENTS = [ + "Energy and supply", + "Network costs", + "Taxes, fees, levies, and charges", +] + +# List of components that add up to the total price. +# NOTE: See find_best_combination_of_components to understand how this choice was made. +COMPONENTS_THAT_ADD_UP_TO_TOTAL = ["Energy and supply", "Network costs", "Taxes, fees, levies, and charges"] + +# Label to use for the calculated total price based on the sum of the main components. +COMPONENTS_TOTAL_PRICE_LABEL = "Total price, including taxes" + + +def sanity_check_inputs(tb: Table) -> None: + # Ensure all relevant dataset codes are present. + error = "Some dataset codes are missing." + assert set(DATASET_CODES_AND_NAMES) <= set(tb["dataset_code"]), error + # Check that each dataset has only one value in fields "freq", "product", and "nrg_cons". + # error = "Some datasets have more than one value in field 'freq'." + # assert (tb.groupby("dataset_code")["freq"].nunique() == 1).all(), error + # error = "Expected 'freq' column to be either A (annual) or S (bi-annual)." + # assert set(tb["freq"].dropna()) == set(["A", "S"]), error + # error = "Some datasets have more than one value in field 'product'." + # assert (tb.dropna(subset="product").groupby("dataset_code")["product"].nunique() == 1).all(), error + # error = "Expected 'product' column to be either 4100 (gas) or 6000 (electricity)." + # assert set(tb["product"].dropna()) == set([4100, 6000]), error + error = "Expected electricity prices to be measured in kWh." + assert set( + tb[tb["dataset_code"].isin(["nrg_pc_204", "nrg_pc_205", "nrg_pc_204_h", "nrg_pc_205_h", "nrg_pc_206_h"])][ + "energy_unit" + ] + ) == set(["KWH"]), error + # error = "Expected 'customer' column to be empty, for the selected datasets." + # assert set(tb["customer"].dropna()) == set(), error + # error = "Expected 'consom' column to be empty, for the selected datasets (that column is only relevant for historical data)." + # assert set(tb["consom"].dropna()) == set(), error + for field, mapping in INDEXES_MAPPING.items(): + if field == "flag": + # Flags need to first be extracted from the value (so they will be sanity checked later). + continue + error = f"Unexpected values in field '{field}'." + assert set(tb[field].dropna()) == set(mapping), error + + +def prepare_inputs(tb: Table) -> Table: + # Values sometimes include a letter, which is a flag. Extract those letters and create a separate column with them. + # Note that sometimes there can be multiple letters (which means multiple flags). + tb["flag"] = tb["value"].astype("string").str.extract(r"([a-z]+)", expand=False) + tb["value"] = tb["value"].str.replace(r"[a-z]", "", regex=True) + + # Some values are start with ':' (namely ':', ': ', ': c', ': u', ': cd'). Replace them with nan. + tb.loc[tb["value"].str.startswith(":"), "value"] = None + + # Assign a proper type to the column of values. + tb["value"] = tb["value"].astype(float) + + # Create a clean column of years, and another of dates. + tb["year-semester"] = tb["year"].str.strip().copy() + tb["year"] = tb["year-semester"].str[0:4].astype(int) + # For the date column: + # * For the first semester, use April 1st. + # * For the second semester, use October 1st. + # * For annual data, use July 1st. + semester_1_mask = tb["year-semester"].str.contains("S1") + semester_2_mask = tb["year-semester"].str.contains("S2") + annual_mask = tb["year-semester"].str.isdigit() + error = "Unexpected values in field 'year-semester'." + assert (semester_1_mask | semester_2_mask | annual_mask).all(), error + tb["date"] = pd.to_datetime(tb["year"].astype(str) + "-07-01") + tb.loc[semester_1_mask, "date"] = pd.to_datetime(tb[semester_1_mask]["year"].astype(str) + "-04-01") + tb.loc[semester_2_mask, "date"] = pd.to_datetime(tb[semester_2_mask]["year"].astype(str) + "-10-01") + + return tb + + +def harmonize_indexes_and_countries(tb: Table) -> Table: + # Add a column with the dataset name. + tb["dataset_name"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODES_AND_NAMES, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + + # Harmonize all other index names. + for field, mapping in INDEXES_MAPPING.items(): + # Avoid categorical dtypes. + tb[field] = tb[field].astype("string") + not_null_mask = tb[field].notnull() + tb.loc[not_null_mask, field] = map_series( + tb[not_null_mask][field], + mapping=mapping, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + + # Harmonize country names. + # Countries are given in NUTS (Nomenclature of Territorial Units for Statistics) codes. + # Region codes are defined in: https://ec.europa.eu/eurostat/web/nuts/correspondence-tables + # There are additional codes not included there, namely: + # EA: Countries in the Euro Area, that use the Euro as their official currency. + # In the historical datasets, there are some additional regions: + # EU15: The 15 countries that made up the EU prior to its 2004 expansion. + # EU25: The 25 member states after the 2004 enlargement, which added ten countries. + # EU27_2007: The 27 EU member states in 2007. + # EU27_2020: The 27 EU members after the United Kingdom left in 2020. + # UA: Ukraine (not a member of the EU, but often included in some European data). + # UK: United Kingdom (not a member since 2020, but included in some European data). + tb = geo.harmonize_countries( + df=tb, countries_file=paths.country_mapping_path, excluded_countries_file=paths.excluded_countries_path + ) + + return tb + + +######################################################################################################################## + +# The code in this block is not used in the data pipeline, but it was useful to understand the data and justify some of the choices. + + +def compare_components_and_prices_data(tb: Table) -> None: + # Compare biannual prices data (without averaging overs semesters) with annual components data. + price_level = "All taxes and levies included" + tb_biannual = tb[(tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_biannual = tb_biannual[(tb_biannual["price_component_or_level"] == price_level)][ + ["dataset_code", "country", "date", "value"] + ] + # Similarly, for annual data, assign July 1st. + tb_annual = tb[(~tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_annual["dataset_code"] = tb_annual["dataset_code"].str.replace("_c", "") + + combination = COMPONENTS_THAT_ADD_UP_TO_TOTAL + annual_components = ( + tb_annual[(tb_annual["price_component_or_level"].isin(combination))] + .groupby( + ["dataset_code", "country", "date"], + observed=True, + as_index=False, + ) + .agg({"value": lambda x: x.sum(min_count=1)}) + .dropna() + .reset_index(drop=True) + ) + + # Combine both datasets for plotting. + compared = pd.concat( + [annual_components.assign(**{"source": "components"}), tb_biannual.assign(**{"source": "prices"})], + ignore_index=True, + ) + # Only a few country-years could be compared this way. Most of the points in the prices datasets were missing. + for dataset_code in compared["dataset_code"].unique(): + for country in sorted(set(compared["country"])): + _compared = compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)] + if len(set(_compared["source"])) < 2: + continue + px.line( + _compared, + x="date", + y="value", + color="source", + markers=True, + title=f"{dataset_code} - {country}", + ).update_yaxes(range=[0, None]).show() + + +def find_best_combination_of_components(tb: Table) -> None: + # Check that the resulting total price for the components dataset (summing up components) is similar to the biannual electricity prices data. + + # Ideally, the prices obtained by adding up components (in the components dataset) should be similar to those obtained in the prices dataset. + # However, both are very sparse (especially the prices dataset), and the prices dataset is also given in semesters, which makes it difficult to compare (without having the actual consumption of each semester to be able to compute a weighted average). + # Transforming biannual data into annual data is not straightforward. + # I tried simply taking the average, but what I found is that the annual components prices (summed over all components) tends to be systematically higher than the biannual prices (averaged over the two semester of the year). I suppose this was caused by doing a simple average instead of weighting by consumption. In semesters with higher consumption (e.g. winter), the increased demand tends to drive prices up. Annual prices, as far as I understand, are consumption-weighted averages, and therefore assign a larger weight to those semesters with higher prices. So, intuitively, it makes sense that the true annual prices tend to be higher than the averaged biannual prices. + # We could create a weighted average, but we would need the actual consumption of each semester (which I haven't found straightaway). + + # Compute an annual average only if there is data for the two semesters. + price_level = "All taxes and levies included" + tb_biannual = tb[(tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_biannual_filtered = ( + tb_biannual.dropna(subset="value") + .groupby( + ["country", "year", "dataset_code", "price_component_or_level"], + observed=True, + as_index=False, + ) + .filter(lambda x: len(x) == 2) + ) + tb_biannual = tb_biannual_filtered.groupby( + ["country", "year", "dataset_code", "price_component_or_level"], + observed=True, + as_index=False, + ).agg({"value": "mean"}) + tb_biannual = tb_biannual[(tb_biannual["price_component_or_level"] == price_level)][ + ["dataset_code", "country", "year", "value"] + ] + # Similarly, for annual data, assign July 1st. + tb_annual = tb[(~tb["year-semester"].str.contains("S")) & (tb["currency"] == "Euro")].reset_index(drop=True) + tb_annual["dataset_code"] = tb_annual["dataset_code"].str.replace("_c", "") + + def _get_annual_sum(tb_annual, combination): + annual_components = ( + tb_annual[(tb_annual["price_component_or_level"].isin(combination))] + .groupby( + ["dataset_code", "country", "year"], + observed=True, + as_index=False, + ) + .agg({"value": lambda x: x.sum(min_count=1)}) + .dropna() + .reset_index(drop=True) + ) + + return annual_components + + import itertools + + from tqdm.auto import tqdm + + # Get all possible combinations of components. + elements = INDEXES_MAPPING["price_component"].values() + combinations = [] + for r in range(1, len(elements) + 1): + combinations.extend(itertools.combinations(elements, r)) + # Keep only combinations that include "Energy and supply" and "Network costs". + combinations = [c for c in combinations if "Energy and supply" in c and "Network costs" in c] + + # Brute-force analysis: Check which combination of components minimizes the error between the sum of components and the prices data. + # NOTE: This takes about 4 minutes. + error_median = [] + error_mean = [] + error_max = [] + for combination in tqdm(combinations): + annual_components = _get_annual_sum(tb_annual, combination) + compared = ( + tb_biannual.merge( + annual_components, + on=["dataset_code", "country", "year"], + how="inner", + suffixes=("_prices", "_components"), + ) + .dropna() + .reset_index(drop=True) + ) + compared["pct"] = 100 * abs(compared["value_prices"] - compared["value_components"]) / compared["value_prices"] + error_mean.append(compared["pct"].mean()) + error_median.append(compared["pct"].median()) + error_max.append(compared["pct"].max()) + # Find the combination that minimizes the error. + results_df = pd.DataFrame( + { + "combination": combinations, + "error_mean": error_mean, + "error_median": error_median, + "error_max": error_max, + } + ) + # There is no single combination that minimizes all error. + set(results_df[results_df["error_mean"] == results_df["error_mean"].min()]["combination"]) + # After inspection, there are different combinations that minimize error (since some components are actually always zero). In terms of the minimum mean error, the combinations are: + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Capacity taxes allowances'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Capacity taxes allowances', 'Nuclear taxes allowance'), + # ('Energy and supply', 'Network costs', 'Taxes, fees, levies, and charges', 'Nuclear taxes allowance') + # Given that some of those allowance components are actually (almost) always zero, it seems clear that + # the best combination is, as expected: + components_optimal = ["Energy and supply", "Network costs", "Taxes, fees, levies, and charges"] + annual_components = _get_annual_sum(tb_annual, combination=components_optimal) + compared = ( + tb_biannual.merge( + annual_components, on=["dataset_code", "country", "year"], how="inner", suffixes=("_prices", "_components") + ) + .dropna() + .reset_index(drop=True) + ) + compared["pct"] = 100 * abs(compared["value_prices"] - compared["value_components"]) / compared["value_prices"] + compared.sort_values("pct", ascending=False).head(60) + # For most countries, the agreement is good, but some country-years, the discrepancy is significant, e.g. Denmark non-household electricity in 2022 and 2023, with an error of 26%. + # There are only a few other discrepancies above 10%. + + # Visually inspect these discrepancies. + compared = pd.concat( + [annual_components.assign(**{"source": "components"}), tb_biannual.assign(**{"source": "prices"})], + ignore_index=True, + ) + # Only a few country-years could be compared this way. Most of the points in the prices datasets were missing. + for dataset_code in compared["dataset_code"].unique(): + for country in sorted(set(compared["country"])): + if ( + len( + set( + compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)][ + "source" + ] + ) + ) + < 2 + ): + continue + px.line( + compared[(compared["dataset_code"] == dataset_code) & (compared["country"] == country)], + x="year", + y="value", + color="source", + markers=True, + title=f"{dataset_code} - {country}", + ).update_yaxes(range=[0, None]).show() + + # Conclusions: + # * The prices and components datasets coincide reasonably well. To recover prices, it seems that the components to be added up are just "Energy and supply", "Network costs", and "Taxes, fees, levies, and charges". For most countries, this combination gives a good agreement with the prices dataset. However, for some countries, there is a significant discrepancy. + # * Numerically, I have checked that for all price components datasets, "Taxes, fees, levies and charges" coincides with the sum of 'Capacity taxes', 'Environmental taxes', 'Nuclear taxes', 'Renewable taxes', 'Value added tax (VAT)', 'Other'. For some country-years, there is a small discrepancy. + # * What's not so clear is what happens with the "allowances". Is "Taxes, fees, levies, and charges allowance" the sum of all other "* allowance"? It's hard to know, since it's non-zero only once (nrg_pc_204_c Netherlands 2023). At that point, it does coincide with the sum of all other "* allowance". But there are other instances of non-zero "* allowance" where "Taxes...allowance" is not defined. It may be possible that allowances are not included in the prices dataset. + + +def plot_final_comparison_between_prices_and_components_data(tb: Table) -> None: + for country in tb["country"].unique(): + for consumer_type in tb["consumer_type"].unique(): + for source in tb["source"].unique(): + _tb = tb[ + (tb["country"] == country) + & (tb["source"] == source) + & (tb["consumer_type"] == consumer_type) + & ( + tb["price_component_or_level"].isin( + [COMPONENTS_TOTAL_PRICE_LABEL, "All taxes and levies included"] + ) + ) + ] + if len(_tb["price_component_or_level"].unique()) < 2: + continue + px.line( + _tb, + x="date", + y="price_euro", + color="price_component_or_level", + markers=True, + title=f"{consumer_type} {source} - {country}", + ).show() + + +######################################################################################################################## + + +def select_and_prepare_relevant_data(tb: Table) -> Table: + # All datasets have a energy unit except electricity components (both for household and non-households). + # I assume the energy unit is kWh. + error = "Expected electricity components (both for household and non-households) to have no energy unit. Remove this code." + assert tb[tb["dataset_code"].isin(["nrg_pc_204_c", "nrg_pc_205_c"])]["energy_unit"].isnull().all(), error + tb.loc[tb["dataset_code"].isin(["nrg_pc_204_c", "nrg_pc_205_c"]), "energy_unit"] = "kWh" + + error = "Expected all datasets to have the same energy unit (kWh)." + assert ( + tb.groupby(["dataset_code"], observed=True, as_index=False) + .agg({"energy_unit": lambda x: "kWh" in x.unique()})["energy_unit"] + .all() + ), error + # Select the same energy unit for all datasets (kWh). + tb = tb[tb["energy_unit"] == "kWh"].drop(columns=["energy_unit"], errors="raise").reset_index(drop=True) + + # Convert prices from price per kWh to price per MWh. + tb["value"] *= 1000 + + # For convenience, instead of having a column for price component (for components datasets) and price level (for prices datasets), create a single column with the price component or level. + assert tb[(tb["price_level"].isnull()) & (tb["price_component"].isnull())].empty + assert tb[(tb["price_level"].notnull()) & (tb["price_component"].notnull())].empty + tb["price_component_or_level"] = tb["price_level"].fillna(tb["price_component"]) + tb = tb.drop(columns=["price_level", "price_component"], errors="raise") + + # After inspection, it looks like the "All bands" consumption is very sparse in the prices datasets. + # One option (if we decided to use the prices dataset) would be to use the more common consumption bands only, which are better informed. + # In the components dataset, "All bands" seems to be less sparse (at least from 2019 onwards). + # To get the total price from the components dataset, we would need to add up components. + # But we would need to figure out which one is the subset of components that ensures no double-counting. + tb = ( + tb[tb["consumption_band"] == "All bands"] + .drop(columns=["consumption_band"], errors="raise") + .reset_index(drop=True) + ) + + # Find the combination of price components that needs to be summed up to recover the full prices. + # NOTE: Uncomment to perform the analysis again, and see conclusions in the following function to understand the choices. + # find_best_combination_of_components(tb=tb) + + # Visually compare the resulting prices obtained by adding up certain components, with the original prices data. + # NOTE: Uncomment to perform some visual checks. + # compare_components_and_prices_data(tb=tb) + + # Remove groups (of country-year-dataset-currency) from the components dataset for which certain components (e.g. "Energy and supply") are not included. + # For example, Albania doesn't have "Energy and supply" costs for household electricity, but it does have other components (e.g. "Network costs"). + tb.loc[ + (tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)) + & ( + ~tb.groupby(["country", "year", "currency"])["price_component_or_level"].transform( + lambda x: all(comp in x.tolist() for comp in MANDATORY_PRICE_COMPONENTS) + ) + ), + "value", + ] = None + + # Remove empty rows. + tb = tb.dropna(subset=["value"]).reset_index(drop=True) + + # Remove data with certain flags. + tb = tb[ + ~tb["flag"].isin( + [ + "confidential", + "definition differs", + "low reliability", + "confidential, definition differs", + "unknown flag", + ] + ) + ].reset_index(drop=True) + error = "Unexpected flag values." + assert set(tb["flag"].dropna()) <= set(["estimated", "break in time series", "provisional", "unknown flag"]), error + + # Add total price to the components dataset, by adding up the contribution of the main components. + tb_components_total = ( + tb[ + tb["dataset_code"].isin(DATASET_CODES_COMPONENTS) + & (tb["price_component_or_level"].isin(COMPONENTS_THAT_ADD_UP_TO_TOTAL)) + ] + .groupby( + ["currency", "country", "year", "dataset_code", "year-semester", "date", "dataset_name"], + observed=True, + as_index=False, + ) + .agg({"value": "sum"}) + .assign(**{"price_component_or_level": COMPONENTS_TOTAL_PRICE_LABEL}) + ) + tb = pr.concat([tb, tb_components_total], ignore_index=True) + + # Create a column for the energy source. + tb["source"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODE_TO_ENERGY_SOURCE, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + error = "Unexpected energy source." + assert set(tb["source"]) == set(["Gas", "Electricity"]), error + + # Create a column for the consumer type. + tb["consumer_type"] = map_series( + tb["dataset_code"], + mapping=DATASET_CODE_TO_CONSUMER_TYPE_MAPPING, + warn_on_missing_mappings=True, + warn_on_unused_mappings=True, + show_full_warning=True, + ) + error = "Unexpected consumer type." + assert set(tb["consumer_type"]) == set(["Household", "Non-household"]), error + + # Drop unnecessary columns. + tb = tb.drop(columns=["flag", "year-semester", "dataset_name"], errors="raise") + + # It would be confusing to keep different national currencies, so, keep only Euro and PPS. + tb = tb[tb["currency"].isin(["Euro", "PPS"])].reset_index(drop=True) + + # Separate euros and PPS in two different columns. + tb = ( + tb[tb["currency"] == "Euro"] + .drop(columns=["currency"]) + .merge( + tb[tb["currency"] == "PPS"].drop(columns=["currency"]), + how="outer", + on=["country", "year", "date", "dataset_code", "source", "price_component_or_level", "consumer_type"], + suffixes=("_euro", "_pps"), + ) + .rename(columns={"value_euro": "price_euro", "value_pps": "price_pps"}, errors="raise") + ) + + return tb + + +def prepare_wide_tables(tb: Table) -> Dict[str, Table]: + # Table for average prices (in euros) of gas and electricity prices of household and non-household consumers. + tb_prices_euro = tb[tb["dataset_code"].isin(DATASET_CODES_PRICES)].pivot( + index=["country", "date"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_euro", + join_column_levels_with="-", + ) + # Table for average prices (in PPS) of gas and electricity prices of household and non-household consumers. + tb_prices_pps = tb[tb["dataset_code"].isin(DATASET_CODES_PRICES)].pivot( + index=["country", "date"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_pps", + join_column_levels_with="-", + ) + # Improve tables format. + tb_prices_euro = tb_prices_euro.format(["country", "date"], short_name="gas_and_electricity_prices_euro_flat") + tb_prices_pps = tb_prices_pps.format(["country", "date"], short_name="gas_and_electricity_prices_pps_flat") + + # Improve column names. + tb_prices_euro = tb_prices_euro.rename( + columns={column: column.replace("__", "_") + "_euro" for column in tb_prices_euro.columns}, errors="raise" + ) + tb_prices_pps = tb_prices_pps.rename( + columns={column: column.replace("__", "_") + "_pps" for column in tb_prices_pps.columns}, errors="raise" + ) + + # Table for price components (in euros) of gas and electricity prices of household and non-household consumers. + tb_price_components_euro = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].pivot( + index=["country", "year"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_euro", + join_column_levels_with="-", + ) + # Table for price components (in PPS) of gas and electricity prices of household and non-household consumers. + tb_price_components_pps = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].pivot( + index=["country", "year"], + columns=["source", "consumer_type", "price_component_or_level"], + values="price_pps", + join_column_levels_with="-", + ) + # Improve tables format. + tb_price_components_euro = tb_price_components_euro.format( + ["country", "year"], short_name="gas_and_electricity_price_components_euro_flat" + ) + tb_price_components_pps = tb_price_components_pps.format( + ["country", "year"], short_name="gas_and_electricity_price_components_pps_flat" + ) + + # Improve column names. + tb_price_components_euro = tb_price_components_euro.rename( + columns={column: column.replace("__", "_") + "_euro" for column in tb_price_components_euro.columns}, + errors="raise", + ) + tb_price_components_pps = tb_price_components_pps.rename( + columns={column: column.replace("__", "_") + "_pps" for column in tb_price_components_pps.columns}, + errors="raise", + ) + + return tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps + + +def sanity_check_outputs(tb: Table) -> None: + error = "Expected 'Energy and supply' and 'Network costs' to be non-negative." + assert tb[ + tb["dataset_code"].isin(DATASET_CODES_COMPONENTS) + & tb["price_component_or_level"].isin(["Energy and supply", "Network costs"]) + & (tb["price_euro"] < 0) + ].empty, error + + # Further sanity checks on component prices. + tb_components = tb[tb["dataset_code"].isin(DATASET_CODES_COMPONENTS)].reset_index(drop=True) + tb_taxes_sum = ( + tb_components[ + tb_components["price_component_or_level"].isin( + [ + "Capacity taxes", + "Environmental taxes", + "Nuclear taxes", + "Renewable taxes", + "Value added tax (VAT)", + "Other", + ] + ) + ] + .groupby(["dataset_code", "country", "year"], observed=True, as_index=False) + .agg({"price_euro": "sum"}) + ) + tb_taxes_original = tb_components[ + tb_components["price_component_or_level"] == "Taxes, fees, levies, and charges" + ].reset_index(drop=True)[["dataset_code", "country", "year", "price_euro"]] + # NOTE: The median value of the sum is 0.0191 euros/kWh. When comparing the percentage difference, ignore values that are too small. + compared = tb_taxes_sum.merge( + tb_taxes_original, how="outer", on=["dataset_code", "country", "year"], suffixes=("_sum", "_original") + ) + compared["dev"] = 100 * ( + abs(compared["price_euro_sum"] - compared["price_euro_original"]) / compared["price_euro_original"] + ) + error = "Expected the sum of 'Capacity taxes', 'Environmental taxes', 'Nuclear taxes', 'Renewable taxes', 'Value added tax (VAT)', 'Other' to coincide with 'Taxes, fees, levies, and charges', within 2% (ignoring any prices below 0.007, which is 17% of rows)." + # NOTE: Some dataset-country-year have a significant discrepancy, e.g. nrg_pc_202_c-Greece-2022, with a price of 6.7€/MWh. + assert compared[(compared["price_euro_original"] > 7) & (compared["dev"] > 2)].empty, error + # compared.sort_values("dev", ascending=False).head(60) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("gas_and_electricity_prices") + + # Read table from meadow dataset. + tb = ds_meadow.read("gas_and_electricity_prices") + + # + # Process data. + # + # Select relevant dataset codes, and add a column with the dataset name. + tb = tb[tb["dataset_code"].isin(DATASET_CODES_AND_NAMES.keys())].reset_index(drop=True) + + # Select and rename columns. + tb = tb[list(COLUMNS)].rename(columns=COLUMNS, errors="raise") + + # Sanity checks on inputs. + sanity_check_inputs(tb=tb) + + # Clean inputs. + tb = prepare_inputs(tb=tb) + + # Harmonize indexes and country names. + tb = harmonize_indexes_and_countries(tb=tb) + + # Select and prepare relevant data. + tb = select_and_prepare_relevant_data(tb=tb) + + # Sanity check outputs. + sanity_check_outputs(tb=tb) + + # Uncomment to plot a comparison (for each country, source, and consumer type) between the prices and the components data. + # NOTE: Some of the biggest discrepancies happen where prices data is given only for one of the semesters. This is the case of Georgia household electricity in 2021 and 2022, where we can't see the value of the missing semester (which could explain why the components data is significantly higher). + # plot_final_comparison_between_prices_and_components_data(tb=tb) + + # Create convenient wide tables. + tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps = prepare_wide_tables(tb=tb) + + # Improve main table format. + tb = tb.drop(columns=["dataset_code"]).format( + ["country", "date", "source", "consumer_type", "price_component_or_level"] + ) + + # + # Save outputs. + # + # Create a new garden dataset. + ds_garden = create_dataset( + dest_dir, + tables=[tb, tb_prices_euro, tb_prices_pps, tb_price_components_euro, tb_price_components_pps], + check_variables_metadata=True, + default_metadata=ds_meadow.metadata, + ) + ds_garden.save() diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json new file mode 100644 index 00000000000..37c3e86047f --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.countries.json @@ -0,0 +1,51 @@ +{ + "Algeria": "Algeria", + "Angola": "Angola", + "Argentina": "Argentina", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Bolivia": "Bolivia", + "Brunei": "Brunei", + "China": "China", + "Colombia": "Colombia", + "Croatia": "Croatia", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "France": "France", + "Gabon": "Gabon", + "Ghana": "Ghana", + "Hungary": "Hungary", + "India": "India", + "Indonesia": "Indonesia", + "Iran": "Iran", + "Iraq": "Iraq", + "Kazakhstan": "Kazakhstan", + "Kuwait": "Kuwait", + "Libya": "Libya", + "Malaysia": "Malaysia", + "Mexico": "Mexico", + "Nigeria": "Nigeria", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Poland": "Poland", + "Qatar": "Qatar", + "Russia": "Russia", + "Slovak Republic": "Slovakia", + "Thailand": "Thailand", + "Turkmenistan": "Turkmenistan", + "UAE": "United Arab Emirates", + "Ukraine": "Ukraine", + "United Kingdom": "United Kingdom", + "Uzbekistan": "Uzbekistan", + "Venezuela": "Venezuela", + "Vietnam": "Vietnam", + "World": "World", + "ElSalvador": "El Salvador", + "SaudiArabia": "Saudi Arabia", + "SouthAfrica": "South Africa", + "SriLanka": "Sri Lanka", + "Taipei": "Taiwan", + "TrinidadandTobago": "Trinidad and Tobago" +} \ No newline at end of file diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml new file mode 100644 index 00000000000..772f0f01622 --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.meta.yml @@ -0,0 +1,45 @@ +definitions: + common: + presentation: + topic_tags: + - Energy + - Fossil Fuels + grapher_config: + note: This data is expressed in constant {dollar_year} US$. + processing_level: minor + unit: constant {dollar_year} US$ + short_unit: "$" + description_short: |- + This data is expressed in US dollars. It is adjusted for inflation but does not account for differences in living costs between countries. + +dataset: + update_period_days: 365 + +tables: + fossil_fuel_subsidies: + variables: + coal_subsidy: + title: Subsidies to coal + electricity_subsidy: + title: Subsidies to electricity + gas_subsidy: + title: Subsidies to gas + oil_subsidy: + title: Subsidies to oil + total_subsidy: + title: Total subsidies + transport_oil_subsidy: + title: Subsidies to oil in transport + subsidization_rate: + title: Subsidization rate + unit: "%" + short_unit: "%" + subsidy_per_capita: + title: Subsidy per capita + unit: constant {dollar_year} US$ per person + short_unit: "$/person" + subsidy_as_share_of_gdp: + title: Subsidy as share of GDP + unit: "%" + short_unit: "%" + diff --git a/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..986ad16ad85 --- /dev/null +++ b/etl/steps/data/garden/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,58 @@ +"""Load a meadow dataset and create a garden dataset.""" + +from etl.data_helpers import geo +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Assumed USD year. +DOLLAR_YEAR = 2023 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load meadow dataset. + ds_meadow = paths.load_dataset("fossil_fuel_subsidies") + + # Read tables from meadow dataset. + tb = ds_meadow.read("fossil_fuel_subsidies") + tb_indicators = ds_meadow.read("fossil_fuel_subsidies_indicators") + tb_transport = ds_meadow.read("fossil_fuel_subsidies_transport_oil") + + # + # Process data. + # + # Convert units from millions of dollars to dollars. + tb["subsidy"] *= 1e6 + + # Transpose table. + tb = tb.pivot(index=["country", "year"], columns="product", values="subsidy", join_column_levels_with="_") + + # Rename conveniently. + tb = tb.rename( + columns={column: f"{column}_subsidy" for column in tb.drop(columns=["country", "year"]).columns}, errors="raise" + ) + + # Harmonize country names. + tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path) + + # Include additional indicators from the other tables. + tb = tb.merge(tb_indicators, on=["country", "year"], how="outer") + tb = tb.merge(tb_transport, on=["country", "year"], how="outer") + + # Improve format. + tb = tb.format() + + # + # Save outputs. + # + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = create_dataset( + dest_dir, tables=[tb], check_variables_metadata=True, yaml_params={"dollar_year": DOLLAR_YEAR} + ) + + # Save changes in the new garden dataset. + ds_garden.save() diff --git a/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py b/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py new file mode 100644 index 00000000000..237959d850e --- /dev/null +++ b/etl/steps/data/grapher/energy/2024-11-20/energy_prices.py @@ -0,0 +1,24 @@ +"""Load garden dataset and create a grapher dataset. + +""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # Load tables from garden dataset. + ds_garden = paths.load_dataset("energy_prices") + tb_annual = ds_garden.read("energy_prices_annual", reset_index=False) + tb_monthly = ds_garden.read("energy_prices_monthly", reset_index=False) + + # Create a new grapher dataset. + dataset = create_dataset( + dest_dir=dest_dir, + tables=[tb_annual, tb_monthly], + check_variables_metadata=True, + default_metadata=ds_garden.metadata, + ) + dataset.save() diff --git a/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..ea4798a2dbd --- /dev/null +++ b/etl/steps/data/grapher/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,26 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("fossil_fuel_subsidies") + + # Read table from garden dataset. + tb = ds_garden.read("fossil_fuel_subsidies", reset_index=False) + + # + # Save outputs. + # + # Create a new grapher dataset. + ds_grapher = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True) + + # Save changes in the new grapher dataset. + ds_grapher.save() diff --git a/etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py b/etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py new file mode 100644 index 00000000000..f8ae1613de9 --- /dev/null +++ b/etl/steps/data/meadow/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -0,0 +1,32 @@ +"""Load a snapshot and create a meadow dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("european_wholesale_electricity_prices.csv") + + # Load data from snapshot. + tb = snap.read() + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format(["country", "date"]) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=snap.metadata) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py b/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..071b4143ba3 --- /dev/null +++ b/etl/steps/data/meadow/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,78 @@ +"""Load a snapshot and create a meadow dataset.""" + +import zipfile + +import owid.catalog.processing as pr + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("gas_and_electricity_prices.zip") + + # Create a list to store each table. + tables = [] + # Open the ZIP file and read each TSV file. + with zipfile.ZipFile(snap.path, "r") as zip_file: + for file_name in zip_file.namelist(): + # Read each TSV file into a table. + with zip_file.open(file_name) as file: + dataset_code = file_name.split(".")[0] + # Each data file starts with comma-separated index columns, followed by tab-separated time data. + # Example: + # freq,product,nrg_cons,unit,tax,currency,geo\TIME_PERIOD 2007 2008... + # And for some datasets, there is annual data, and for others bi-annual data, e.g. 2007-S1 2007-S2 2008-S1... + # First, load this file as a table. + _tb = pr.read_csv( + file, sep=r",|\t", engine="python", metadata=snap.to_table_metadata(), origin=snap.metadata.origin + ) + # Identify index columns. + index_columns = [column for column in _tb.columns if not column[0].isdigit()] + # Melt the table to have a single "time" column. + _tb = _tb.melt(id_vars=index_columns, var_name="time", value_name="value") + # Remove spurious "TIME_PERIOD" from one of the columns. + _tb = _tb.rename(columns={column: column.replace("\\TIME_PERIOD", "") for column in _tb.columns}) + # Add the dataset code as a column. + _tb = _tb.assign(**{"dataset_code": dataset_code}) + # Append current table to the list. + tables.append(_tb) + + # Concatenate all tables. + tb = pr.concat(tables, ignore_index=True) + + # + # Process data. + # + # Ensure all columns are snake-case, set an appropriate index, and sort conveniently. + tb = tb.format( + [ + "freq", + "product", + "nrg_cons", + "unit", + "tax", + "currency", + "geo", + "time", + "dataset_code", + "nrg_prc", + "customer", + "consom", + ] + ) + + # + # Save outputs. + # + # Create a new meadow dataset with the same metadata as the snapshot. + ds_meadow = create_dataset(dest_dir, tables=[tb], check_variables_metadata=True) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py b/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..85eb9491ead --- /dev/null +++ b/etl/steps/data/meadow/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,128 @@ +"""Load a snapshot and create a meadow dataset.""" + +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + +# Assumed USD year. +DOLLAR_YEAR = 2023 + + +def prepare_subsidies_by_country_table(tb_subsidies: Table) -> Table: + # The table is split into two subtables: Above, global data, and below, country data. They are separated by an empty row. + columns_total = tb_subsidies.loc[0].tolist() + table_country_start_index = tb_subsidies[tb_subsidies[tb_subsidies.columns[0]] == "Country"].index[0] + columns_countries = tb_subsidies.loc[table_country_start_index].tolist() + error = "Subsidies by country sheet has changed." + assert tb_subsidies.columns[0] == f"Unit: Real {DOLLAR_YEAR} million USD", error + # Check that tables are aligned. + assert columns_total[2:] == columns_countries[2:], error + # Rename columns. + columns = columns_countries[0:2] + [str(int(year)) for year in columns_countries[2:]] + # Extract global subtable and assign country "World". + tb_global = tb_subsidies.loc[1 : table_country_start_index - 1].dropna(how="all").reset_index(drop=True) + tb_global = tb_global.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_global.columns, columns)}, errors="raise" + ) + tb_global["Country"] = "World" + tb_global["Product"] = tb_global["Product"].replace({"All Products": "Total", "Natural Gas": "Gas"}) + # Extract countries subtable. + tb_countries = tb_subsidies.loc[table_country_start_index + 1 :].reset_index(drop=True) + tb_countries = tb_countries.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_countries.columns, columns)}, + errors="raise", + ) + # Combine both tables. + tb = pr.concat([tb_global, tb_countries], ignore_index=True) + # Transpose table. + tb = tb.melt(id_vars=["Country", "Product"], var_name="Year", value_name="subsidy") + + # Improve format. + tb = tb.format(["country", "year", "product"]) + + return tb + + +def prepare_indicators_by_country_table(tb_indicators: Table) -> Table: + # The year of the data is given in the very first cell. The actual table starts a few rows below. + error = "Indicators by country sheet has changed." + assert tb_indicators.columns[0] == f"Indicators for year {DOLLAR_YEAR}", error + columns = { + "Country": "country", + "Average subsidisation rate (%)": "subsidization_rate", + "Subsidy per capita ($/person)": "subsidy_per_capita", + "Total subsidy as share of GDP (%)": "subsidy_as_share_of_gdp", + } + assert tb_indicators.loc[2].tolist() == list(columns), error + tb_indicators = tb_indicators.loc[3:].reset_index(drop=True) + tb_indicators = tb_indicators.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_indicators.columns, columns.values())}, + errors="raise", + ) + # Add a year column. + tb_indicators = tb_indicators.assign(**{"year": DOLLAR_YEAR}) + # Improve format. + tb_indicators = tb_indicators.format(short_name="fossil_fuel_subsidies_indicators") + + return tb_indicators + + +def prepare_transport_oil_table(tb_transport: Table) -> Table: + columns = ["country"] + [str(int(year)) for year in tb_transport.loc[0][1:].tolist()] + error = "Transport Oil Subsidies sheet has changed." + assert tb_transport.columns[0] == f"Unit: Real {DOLLAR_YEAR} million USD", error + assert [column.isdigit() for column in columns[1:]], error + tb_transport = tb_transport.loc[1:].reset_index(drop=True) + tb_transport = tb_transport.rename( + columns={old_column: new_column for old_column, new_column in zip(tb_transport.columns, columns)}, + errors="raise", + ) + # Transpose table. + tb_transport = tb_transport.melt(id_vars=["country"], var_name="year", value_name="transport_oil_subsidy") + # Improve format. + tb_transport = tb_transport.format(short_name="fossil_fuel_subsidies_transport_oil") + + return tb_transport + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Retrieve snapshot. + snap = paths.load_snapshot("fossil_fuel_subsidies.xlsx") + + # Load data from all relevant sheets in the snapshot file. + tb_subsidies = snap.read(sheet_name="Subsidies by country", skiprows=3) + tb_indicators = snap.read(sheet_name="Indicators by country") + tb_transport = snap.read(sheet_name="Transport Oil Subsidies", skiprows=3) + + # + # Process data. + # + # Prepare "Subsidies by country" table. + tb_subsidies = prepare_subsidies_by_country_table(tb_subsidies=tb_subsidies) + + # Prepare "Indicators by country" table. + tb_indicators = prepare_indicators_by_country_table(tb_indicators=tb_indicators) + + # Prepare "Transport Oil Subsidies" table. + tb_transport = prepare_transport_oil_table(tb_transport=tb_transport) + + # + # Save outputs. + # + # Create a new meadow dataset. + ds_meadow = create_dataset( + dest_dir, + tables=[tb_subsidies, tb_indicators, tb_transport], + check_variables_metadata=True, + default_metadata=snap.metadata, + ) + + # Save changes in the new meadow dataset. + ds_meadow.save() diff --git a/etl/steps/export/multidim/energy/latest/energy_prices.py b/etl/steps/export/multidim/energy/latest/energy_prices.py new file mode 100644 index 00000000000..5a9a6e59e08 --- /dev/null +++ b/etl/steps/export/multidim/energy/latest/energy_prices.py @@ -0,0 +1,47 @@ +from etl import multidim +from etl.db import get_engine +from etl.helpers import PathFinder + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load Eurostat data on gas and electricity prices. + ds_grapher = paths.load_dataset("energy_prices") + + # Read table of prices in euros. + tb_annual = ds_grapher.read("energy_prices_annual") + tb_monthly = ds_grapher.read("energy_prices_monthly") + + # + # Process data. + # + # Load configuration from adjacent yaml file. + config = paths.load_mdim_config() + + # Create views. + config["views"] = multidim.generate_views_for_dimensions( + dimensions=config["dimensions"], + tables=[tb_annual, tb_monthly], + dimensions_order_in_slug=("frequency", "source", "consumer", "price_component", "unit"), + warn_on_missing_combinations=False, + additional_config={ + "$schema": "https://files.ourworldindata.org/schemas/grapher-schema.005.json", + "chartTypes": ["LineChart"], + "hasMapTab": True, + "tab": "map", + "map": { + "projection": "Europe", + "colorScale": {"baseColorScheme": "YlOrBr"}, + }, + }, + ) + + # + # Save outputs. + # + multidim.upsert_multidim_data_page(slug="mdd-energy-prices", config=config, engine=get_engine()) diff --git a/etl/steps/export/multidim/energy/latest/energy_prices.yml b/etl/steps/export/multidim/energy/latest/energy_prices.yml new file mode 100644 index 00000000000..f2640249e48 --- /dev/null +++ b/etl/steps/export/multidim/energy/latest/energy_prices.yml @@ -0,0 +1,102 @@ +title: + title: "Energy prices" + titleVariant: "by energy source" +defaultSelection: + - "European Union (27)" +topicTags: + - "Energy" +dimensions: + - slug: "frequency" + name: "Frequency" + choices: + - slug: "annual" + name: "Annual" + description: "Annual data" + - slug: "monthly" + name: "Monthly" + description: "Monthly data" + - slug: "source" + name: "Energy source" + choices: + - slug: "electricity" + name: "Electricity" + - slug: "gas" + name: "Gas" + - slug: "price_component" + name: "Price component" + choices: + - slug: "total_price_including_taxes" + name: "Total consumer price" + description: "Total consumer price including all taxes and levies" + group: Overview + - slug: "wholesale" + name: "Wholesale price" + description: "Wholesale price" + group: Overview + - slug: "energy_and_supply" + name: "Energy and supply" + description: "Energy and supply" + group: Individual price components + - slug: "network_costs" + name: "Network costs" + description: "Network costs" + group: Individual price components + - slug: "capacity_taxes" + name: "Capacity taxes" + description: "Capacity taxes" + group: Individual price components + - slug: "value_added_tax_vat" + name: "Value added tax (VAT)" + description: "Value added tax (VAT)" + group: Individual price components + - slug: "environmental_taxes" + name: "Environmental taxes" + description: "Environmental taxes" + group: Individual price components + - slug: "nuclear_taxes" + name: "Nuclear taxes" + description: "Nuclear taxes" + group: Individual price components + - slug: "renewable_taxes" + name: "Renewable taxes" + description: "Renewable taxes" + group: Individual price components + - slug: "taxes_fees_levies_and_charges" + name: "All taxes, fees, levies and charges" + description: "All taxes, fees, levies and charges" + group: Individual price components + - slug: "other" + name: "Other costs" + description: "Other costs" + group: Individual price components + # Other available price components: + # 'capacity_taxes_allowances', + # 'environmental_taxes_allowance', + # 'nuclear_taxes_allowance', + # 'renewable_taxes_allowance', + # 'taxes_fees_levies_and_charges_allowance', + # 'other_allowance', + - slug: "consumer" + name: "Consumer type" + choices: + - slug: "household" + name: "Households" + description: "Household consumers" + - slug: "non_household" + name: "Non-households" + description: "Non-household consumers" + - slug: "all" + name: "All consumers" + description: "All consumers" + - slug: "unit" + name: "Unit" + choices: + - slug: "euro" + name: "Euro" + description: "Price in euros" + - slug: "pps" + name: "PPS" + description: "Price in Purchasing Power Standard" +views: + # Views will be filled out programmatically. + [] diff --git a/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc new file mode 100644 index 00000000000..ba348bf3808 --- /dev/null +++ b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.csv.dvc @@ -0,0 +1,30 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: European Wholesale Electricity Price Data + description: |- + Wholesale day-ahead electricity price data for European countries. + date_published: "2024-11-20" + title_snapshot: European Wholesale Electricity Price Data - Monthly + + # Citation + producer: Ember + citation_full: |- + Ember - European Wholesale Electricity Price Data. Based on data from European Network of Transmission System Operators (ENTSO-E). + + # Files + url_main: https://ember-energy.org/data/european-wholesale-electricity-price-data/ + url_download: https://storage.googleapis.com/emb-prod-bkt-publicdata/public-downloads/european_wholesale_electricity_price_data_monthly.csv + date_accessed: 2024-11-20 + + # License + license: + name: CC BY 4.0 + url: https://ember-energy.org/creative-commons/ + +outs: + - md5: ce52ff862b464953c93b1f04531d0db5 + size: 94675 + path: european_wholesale_electricity_prices.csv diff --git a/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py new file mode 100644 index 00000000000..2f52db1b75d --- /dev/null +++ b/snapshots/ember/2024-11-20/european_wholesale_electricity_prices.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of dataset.""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"ember/{SNAPSHOT_VERSION}/european_wholesale_electricity_prices.csv") + + # Download data from source, add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py new file mode 100644 index 00000000000..b0bf9e6d6af --- /dev/null +++ b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.py @@ -0,0 +1,90 @@ +"""Script to create a snapshot of dataset.""" + +import zipfile +from pathlib import Path + +import click +import requests +from tqdm.auto import tqdm + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + +# Base URL for Eurostat API energy data. +BASE_URL = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/" + +# List of dataset codes to download. +URL_DOWNLOADS = [ + #################################################################################################################### + # Energy statistics - natural gas and electricity prices (from 2007 onwards) (nrg_pc) + # Gas prices for household consumers - bi-annual data (from 2007 onwards) (nrg_pc_202) + "nrg_pc_202", + # Gas prices for non-household consumers - bi-annual data (from 2007 onwards) (nrg_pc_203) + "nrg_pc_203", + # Electricity prices for household consumers - bi-annual data (from 2007 onwards) (nrg_pc_204) + "nrg_pc_204", + # Electricity prices for non-household consumers - bi-annual data (from 2007 onwards) (nrg_pc_205) + "nrg_pc_205", + # Household consumption volumes of gas by consumption bands (nrg_pc_202_v) + "nrg_pc_202_v", + # Non-household consumption volumes of gas by consumption bands (nrg_pc_203_v) + "nrg_pc_203_v", + # Household consumption volumes of electricity by consumption bands (nrg_pc_204_v) + "nrg_pc_204_v", + # Non-household consumption volumes of electricity by consumption bands (nrg_pc_205_v) + "nrg_pc_205_v", + # Gas prices components for household consumers - annual data (nrg_pc_202_c) + "nrg_pc_202_c", + # Gas prices components for non-household consumers - annual data (nrg_pc_203_c) + "nrg_pc_203_c", + # Electricity prices components for household consumers - annual data (from 2007 onwards) (nrg_pc_204_c) + "nrg_pc_204_c", + # Electricity prices components for non-household consumers - annual data (from 2007 onwards) (nrg_pc_205_c) + "nrg_pc_205_c", + # Share for transmission and distribution in the network cost for gas and electricity - annual data (nrg_pc_206) + "nrg_pc_206", + #################################################################################################################### + # Energy statistics - natural gas and electricity prices (until 2007) (nrg_pc_h) + # Gas prices for domestic consumers - bi-annual data (until 2007) (nrg_pc_202_h) + "nrg_pc_202_h", + # Gas prices for industrial consumers - bi-annual data (until 2007) (nrg_pc_203_h) + "nrg_pc_203_h", + # Electricity prices for domestic consumers - bi-annual data (until 2007) (nrg_pc_204_h) + "nrg_pc_204_h", + # Electricity prices for industrial consumers - bi-annual data (until 2007) (nrg_pc_205_h) + "nrg_pc_205_h", + # Electricity - marker prices - bi-annual data (until 2007) (nrg_pc_206_h) + "nrg_pc_206_h", + #################################################################################################################### +] +# Further API parameters to download each file. +URL_SUFFIX = "?format=TSV&compressed=false" + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"eurostat/{SNAPSHOT_VERSION}/gas_and_electricity_prices.zip") + + # Ensure output snapshot folder exists, otherwise create it. + snap.path.parent.mkdir(exist_ok=True, parents=True) + + # Create a temporary ZIP file. + with zipfile.ZipFile(snap.path, "w") as zip_file: + # Fetch all relevant datasets from Eurostat API. + for code in tqdm(URL_DOWNLOADS): + # Request the data file for the current dataset. + response = requests.get(f"{BASE_URL}{code}{URL_SUFFIX}") + # Save each file inside the ZIP file. + file_name = f"{code}.tsv" + zip_file.writestr(file_name, response.text) + + # Create snapshot and upload to R2. + snap.create_snapshot(upload=upload, filename=snap.path) + + +if __name__ == "__main__": + main() diff --git a/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc new file mode 100644 index 00000000000..96d2eb2963a --- /dev/null +++ b/snapshots/eurostat/2024-11-05/gas_and_electricity_prices.zip.dvc @@ -0,0 +1,26 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Energy statistics, prices of natural gas and electricity + date_published: "2024-11-04" + + # Citation + producer: Eurostat + citation_full: |- + Eurostat - Energy statistics, prices of natural gas and electricity (2024). + + # Files + url_main: https://ec.europa.eu/eurostat/web/energy/database + date_accessed: 2024-11-05 + + # License + license: + name: CC BY 4.0 + url: https://ec.europa.eu/eurostat/web/main/help/copyright-notice + +outs: + - md5: 285787159fb3f598db8c0fb0ba67eb2c + size: 8286547 + path: gas_and_electricity_prices.zip diff --git a/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py new file mode 100644 index 00000000000..5cb7627fb7c --- /dev/null +++ b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.py @@ -0,0 +1,32 @@ +"""Script to create a snapshot of dataset. + +To obtain the file, you need to log in into the IEA website and download the XLSX file in: +https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database#data-sets + +Note that creating an account is free, and this dataset is also free of cost. + +""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +@click.option("--path-to-file", prompt=True, type=str, help="Path to local data file.") +def main(path_to_file: str, upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"iea/{SNAPSHOT_VERSION}/fossil_fuel_subsidies.xlsx") + + # Copy local data file to snapshots data folder, add file to DVC and upload to S3. + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc new file mode 100644 index 00000000000..a88b8b6e19d --- /dev/null +++ b/snapshots/iea/2024-11-20/fossil_fuel_subsidies.xlsx.dvc @@ -0,0 +1,29 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: Fossil Fuel Subsidies Database + description: |- + Fossil fuel consumption subsidies for selected countries. + date_published: "2024-10-01" + + # Citation + producer: International Energy Agency + citation_full: |- + International Energy Agency - Fossil Fuel Subsidies Database (2024). + attribution_short: IEA + + # Files + url_main: https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database + date_accessed: 2024-11-20 + + # License + license: + name: CC BY 4.0 + url: https://www.iea.org/data-and-statistics/data-product/fossil-fuel-subsidies-database + +outs: + - md5: baa1ef5e6f740931575c19082d28f745 + size: 148786 + path: fossil_fuel_subsidies.xlsx