feat(datasource): Checkbox for always filtering main dttm in datasource

superset-frontend/src/components/Datasource/DatasourceEditor.jsx

@@ -766,6 +766,7 @@ class DatasourceEditor extends React.PureComponent {
         ? encodeURIComponent(datasource.table_name)
         : datasource.table_name,
       normalize_columns: datasource.normalize_columns,
+      time_secondary_column: datasource.time_secondary_column,
     };
     Object.entries(params).forEach(([key, value]) => {
       // rison can't encode the undefined value
@@ -1003,6 +1004,15 @@ class DatasourceEditor extends React.PureComponent {
           )}
           control={<CheckboxControl controlId="normalize_columns" />}
         />
+        <Field
+          inline
+          fieldKey="time_secondary_column"
+          label={t('Time secondary column')}
+          description={t(
+            `When the secondary datetime columns are filtered, apply the same filter to the main datetime column.`,
+          )}
+          control={<CheckboxControl controlId="time_secondary_column" />}
+        />
       </Fieldset>
     );
   }

superset-frontend/src/components/Datasource/DatasourceModal.tsx

@@ -129,6 +129,7 @@ const DatasourceModal: FunctionComponent<DatasourceModalProps> = ({
         description: currentDatasource.description,
         main_dttm_col: currentDatasource.main_dttm_col,
         normalize_columns: currentDatasource.normalize_columns,
+        time_secondary_column: currentDatasource.time_secondary_column,
         offset: currentDatasource.offset,
         default_endpoint: currentDatasource.default_endpoint,
         cache_timeout:

superset-frontend/src/features/datasets/types.ts

@@ -64,4 +64,5 @@ export type DatasetObject = {
   extra?: string;
   is_managed_externally: boolean;
   normalize_columns: boolean;
+  time_secondary_column: boolean;
 };

superset/connectors/sqla/models.py

@@ -545,6 +545,7 @@ class SqlaTable(
     template_params = Column(Text)
     extra = Column(Text)
     normalize_columns = Column(Boolean, default=False)
+    time_secondary_column = Column(Boolean, default=False)
 
     baselink = "tablemodelview"
 
@@ -564,6 +565,7 @@ class SqlaTable(
         "fetch_values_predicate",
         "extra",
         "normalize_columns",
+        "time_secondary_column",
     ]
     update_from_object_fields = [f for f in export_fields if f != "database_id"]
     export_parent = "database"
@@ -761,6 +763,8 @@ def data(self) -> dict[str, Any]:
             data_["health_check_message"] = self.health_check_message
             data_["extra"] = self.extra
             data_["owners"] = self.owners_data
+            data_["time_secondary_column"] = self.time_secondary_column
+            data_["normalize_columns"] = self.normalize_columns
         return data_
 
     @property

superset/connectors/sqla/views.py

@@ -314,6 +314,7 @@ class TableModelView(  # pylint: disable=too-many-ancestors
         "template_params",
         "extra",
         "normalize_columns",
+        "time_secondary_column",
     ]
     base_filters = [["id", DatasourceFilter, lambda: []]]
     show_columns = edit_columns + ["perm", "slices"]
@@ -384,6 +385,12 @@ class TableModelView(  # pylint: disable=too-many-ancestors
             "Allow column names to be changed to case insensitive format, "
             "if supported (e.g. Oracle, Snowflake)."
         ),
+        "time_secondary_column": _(
+            "Datasets can have a main datatime column (main_dttm_col), "
+            "but can also have secondary time columns. "
+            "When this attribute is true, whenever the secondary columns are filtered, "
+            "the same filter is applied to the main datetime column."
+        ),
     }
     label_columns = {
         "slices": _("Associated Charts"),

superset/dashboards/schemas.py

@@ -245,6 +245,7 @@ class DashboardDatasetSchema(Schema):
     time_grain_sqla = fields.List(fields.List(fields.Str()))
     granularity_sqla = fields.List(fields.List(fields.Str()))
     normalize_columns = fields.Bool()
+    time_secondary_column = fields.Bool()
 
 
 class BaseDashboardSchema(Schema):

superset/datasets/api.py

@@ -143,6 +143,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         "description",
         "main_dttm_col",
         "normalize_columns",
+        "time_secondary_column",
         "offset",
         "default_endpoint",
         "cache_timeout",
@@ -221,6 +222,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         "description",
         "main_dttm_col",
         "normalize_columns",
+        "time_secondary_column",
         "offset",
         "default_endpoint",
         "cache_timeout",

superset/datasets/commands/duplicate.py

@@ -68,6 +68,7 @@ def run(self) -> Model:
             table.schema = self._base_model.schema
             table.template_params = self._base_model.template_params
             table.normalize_columns = self._base_model.normalize_columns
+            table.time_secondary_column = self._base_model.time_secondary_column
             table.is_sqllab_view = True
             table.sql = ParsedQuery(self._base_model.sql).stripped()
             db.session.add(table)

superset/datasets/schemas.py

@@ -99,6 +99,7 @@ class DatasetPostSchema(Schema):
     is_managed_externally = fields.Boolean(allow_none=True, dump_default=False)
     external_url = fields.String(allow_none=True)
     normalize_columns = fields.Boolean(load_default=False)
+    time_secondary_column = fields.Boolean(load_default=False)
 
 
 class DatasetPutSchema(Schema):
@@ -111,6 +112,7 @@ class DatasetPutSchema(Schema):
     description = fields.String(allow_none=True)
     main_dttm_col = fields.String(allow_none=True)
     normalize_columns = fields.Boolean(allow_none=True, dump_default=False)
+    time_secondary_column = fields.Boolean(load_default=False)
     offset = fields.Integer(allow_none=True)
     default_endpoint = fields.String(allow_none=True)
     cache_timeout = fields.Integer(allow_none=True)
@@ -250,6 +252,7 @@ def fix_extra(self, data: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
     is_managed_externally = fields.Boolean(allow_none=True, dump_default=False)
     external_url = fields.String(allow_none=True)
     normalize_columns = fields.Boolean(load_default=False)
+    time_secondary_column = fields.Boolean(load_default=False)
 
 
 class GetOrCreateDatasetSchema(Schema):
@@ -266,6 +269,7 @@ class GetOrCreateDatasetSchema(Schema):
         metadata={"description": "Template params for the table"}
     )
     normalize_columns = fields.Boolean(load_default=False)
+    time_secondary_column = fields.Boolean(load_default=False)
 
 
 class DatasetSchema(SQLAlchemyAutoSchema):

superset/db_engine_specs/base.py

@@ -326,7 +326,6 @@ class BaseEngineSpec:  # pylint: disable=too-many-public-methods
     # Does database support join-free timeslot grouping
     time_groupby_inline = False
     limit_method = LimitMethod.FORCE_LIMIT
-    time_secondary_columns = False
     allows_joins = True
     allows_subqueries = True
     allows_alias_in_select = True

superset/db_engine_specs/clickhouse.py

@@ -52,7 +52,6 @@
 class ClickHouseBaseEngineSpec(BaseEngineSpec):
     """Shared engine spec for ClickHouse."""
 
-    time_secondary_columns = True
     time_groupby_inline = True
 
     _time_grain_expressions = {

superset/db_engine_specs/elasticsearch.py

@@ -36,7 +36,6 @@ class ElasticSearchEngineSpec(BaseEngineSpec):  # pylint: disable=abstract-metho
     engine = "elasticsearch"
     engine_name = "ElasticSearch (SQL API)"
     time_groupby_inline = True
-    time_secondary_columns = True
     allows_joins = False
     allows_subqueries = True
     allows_sql_comments = False
@@ -98,7 +97,6 @@ def convert_dttm(
 
 class OpenDistroEngineSpec(BaseEngineSpec):  # pylint: disable=abstract-method
     time_groupby_inline = True
-    time_secondary_columns = True
     allows_joins = False
     allows_subqueries = True
     allows_sql_comments = False

superset/db_engine_specs/kusto.py

@@ -37,7 +37,6 @@ class KustoSqlEngineSpec(BaseEngineSpec):  # pylint: disable=abstract-method
     engine = "kustosql"
     engine_name = "KustoSQL"
     time_groupby_inline = True
-    time_secondary_columns = True
     allows_joins = True
     allows_subqueries = True
     allows_sql_comments = False
@@ -116,7 +115,6 @@ class KustoKqlEngineSpec(BaseEngineSpec):  # pylint: disable=abstract-method
     engine = "kustokql"
     engine_name = "KustoKQL"
     time_groupby_inline = True
-    time_secondary_columns = True
     allows_joins = True
     allows_subqueries = True
     allows_sql_comments = False

superset/db_engine_specs/lib.py

@@ -125,7 +125,6 @@ def diagnose(spec: type[BaseEngineSpec]) -> dict[str, Any]:
             "subqueries": spec.allows_subqueries,
             "alias_in_select": spec.allows_alias_in_select,
             "alias_in_orderby": spec.allows_alias_in_orderby,
-            "secondary_time_columns": spec.time_secondary_columns,
             "time_groupby_inline": spec.time_groupby_inline,
             "alias_to_source_column": not spec.allows_alias_to_source_column,
             "order_by_not_in_select": spec.allows_hidden_orderby_agg,

superset/db_engine_specs/solr.py

@@ -25,7 +25,6 @@ class SolrEngineSpec(BaseEngineSpec):  # pylint: disable=abstract-method
     engine_name = "Apache Solr"
 
     time_groupby_inline = False
-    time_secondary_columns = False
     allows_joins = False
     allows_subqueries = False
 

superset/migrations/versions/2023-09-06_13-18_317970b4400c_added_time_secondary_column_to_.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Added time_secondary_column to datasource
+
+Revision ID: 317970b4400c
+Revises: ec54aca4c8a2
+Create Date: 2023-09-06 13:18:59.597259
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "317970b4400c"
+down_revision = "ec54aca4c8a2"
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import Session
+
+from superset import db
+from superset.migrations.shared.utils import paginated_update
+
+Base = declarative_base()
+
+
+class SqlaTable(Base):
+    __tablename__ = "tables"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    time_secondary_column = sa.Column(sa.Boolean())
+
+
+def upgrade():
+    op.add_column(
+        "tables",
+        sa.Column(
+            "time_secondary_column",
+            sa.Boolean(),
+            nullable=True,
+            default=False,
+            server_default=sa.false(),
+        ),
+    )
+
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    for table in paginated_update(session.query(SqlaTable)):
+        table.time_secondary_column = False
+
+
+def downgrade():
+    op.drop_column("tables", "time_secondary_column")

superset/models/helpers.py

@@ -751,6 +751,10 @@ def offset(self) -> int:
     def main_dttm_col(self) -> Optional[str]:
         raise NotImplementedError()
 
+    @property
+    def time_secondary_column(self) -> Optional[bool]:
+        return False
+
     @property
     def dttm_cols(self) -> list[str]:
         raise NotImplementedError()
@@ -1670,7 +1674,7 @@ def get_sqla_query(  # pylint: disable=too-many-arguments,too-many-locals,too-ma
 
             # Use main dttm column to support index with secondary dttm columns.
             if (
-                db_engine_spec.time_secondary_columns
+                self.time_secondary_column
                 and self.main_dttm_col in self.dttm_cols
                 and self.main_dttm_col != dttm_col.column_name
             ):

superset/views/datasource/schemas.py

@@ -29,6 +29,7 @@ class ExternalMetadataParams(TypedDict):
     schema_name: str
     table_name: str
     normalize_columns: Optional[bool]
+    time_secondary_column: Optional[bool]
 
 
 get_external_metadata_schema = {
@@ -37,6 +38,7 @@ class ExternalMetadataParams(TypedDict):
     "schema_name": "string",
     "table_name": "string",
     "normalize_columns": "boolean",
+    "time_secondary_column": "boolean",
 }
 
 
@@ -46,6 +48,7 @@ class ExternalMetadataSchema(Schema):
     schema_name = fields.Str(allow_none=True)
     table_name = fields.Str(required=True)
     normalize_columns = fields.Bool(allow_none=True)
+    time_secondary_column = fields.Bool(allow_none=True)
 
     # pylint: disable=unused-argument
     @post_load
@@ -60,6 +63,7 @@ def normalize(
             schema_name=data.get("schema_name", ""),
             table_name=data["table_name"],
             normalize_columns=data["normalize_columns"],
+            time_secondary_column=data["time_secondary_column"],
         )
 
 

superset/views/datasource/views.py

@@ -77,7 +77,9 @@ def save(self) -> FlaskResponse:
 
         datasource_dict = json.loads(data)
         normalize_columns = datasource_dict.get("normalize_columns", False)
+        time_secondary_column = datasource_dict.get("time_secondary_column", False)
         datasource_dict["normalize_columns"] = normalize_columns
+        datasource_dict["time_secondary_column"] = time_secondary_column
         datasource_id = datasource_dict.get("id")
         datasource_type = datasource_dict.get("type")
         database_id = datasource_dict["database"].get("id")

tests/integration_tests/dashboard_utils.py

@@ -53,7 +53,12 @@ def create_table_metadata(
 
     table = get_table(table_name, database, schema)
     if not table:
-        table = SqlaTable(schema=schema, table_name=table_name, normalize_columns=False)
+        table = SqlaTable(
+            schema=schema,
+            table_name=table_name,
+            normalize_columns=False,
+            time_secondary_column=False,
+        )
     if fetch_values_predicate:
         table.fetch_values_predicate = fetch_values_predicate
     table.database = database

tests/integration_tests/datasets/api_tests.py

@@ -582,6 +582,7 @@ def test_create_dataset_item_normalize(self):
             "schema": None,
             "table_name": "ab_permission",
             "normalize_columns": True,
+            "time_secondary_column": False,
         }
         uri = "api/v1/dataset/"
         rv = self.post_assert_metric(uri, table_data, "post")

tests/integration_tests/datasets/commands_tests.py

@@ -171,6 +171,7 @@ def test_export_dataset_command(self, mock_g):
                 },
             ],
             "normalize_columns": False,
+            "time_secondary_column": False,
             "offset": 0,
             "params": None,
             "schema": get_example_default_schema(),
@@ -231,6 +232,7 @@ def test_export_dataset_command_key_order(self, mock_g):
             "fetch_values_predicate",
             "extra",
             "normalize_columns",
+            "time_secondary_column",
             "uuid",
             "metrics",
             "columns",