From 5a9eb34a7cab5cbaa5c0ed0830013bbccbf5d216 Mon Sep 17 00:00:00 2001 From: Jan Kadlec Date: Tue, 3 Mar 2026 16:29:11 +0100 Subject: [PATCH 1/2] chore: apply UP045 rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert Optional[X] to `X | None`. Also, remove UP038 rule – there was a warning about removal. JIRA: TRIVIAL risk: low --- .../src/gooddata_dbt/dbt/metrics.py | 13 ++-- .../src/gooddata_dbt/dbt/profiles.py | 6 +- .../src/gooddata_dbt/dbt/tables.py | 38 +++++------ .../src/gooddata_dbt/dbt_plugin.py | 9 ++- .../src/gooddata_dbt/gooddata/config.py | 7 +- .../src/gooddata_dbt/sdk_wrapper.py | 7 +- .../tests/resources/dbt_target/manifest.json | 14 ++++ .../gooddata_layouts/pdm/airports.yaml | 3 + packages/gooddata-dbt/tests/test_tables.py | 28 ++++++++ .../src/gooddata_fdw/column_utils.py | 4 +- .../src/gooddata_fdw/environment.py | 6 +- .../gooddata-fdw/src/gooddata_fdw/executor.py | 10 +-- packages/gooddata-fdw/src/gooddata_fdw/fdw.py | 6 +- .../src/gooddata_fdw/import_workspace.py | 8 +-- .../gooddata-fdw/src/gooddata_fdw/naming.py | 4 +- .../gooddata-fdw/src/gooddata_fdw/options.py | 6 +- .../function/data_source_messages.py | 6 +- .../function/execution_context.py | 66 +++++++++---------- .../function/flight_methods.py | 11 ++-- .../gooddata_flexconnect/function/function.py | 9 ++- .../function/function_invocation.py | 4 +- .../function/function_task.py | 8 +-- .../test_flex_fun_execution_context.py | 1 - .../tests/function/testing_funs.py | 5 +- .../tests/server/funs/fun1.py | 3 +- .../tests/server/funs/fun2.py | 5 +- .../tests/server/funs/fun3.py | 5 +- .../tests/server/funs/fun4.py | 5 +- .../src/gooddata_flight_server/cli.py | 4 +- .../gooddata_flight_server/config/config.py | 32 ++++----- .../errors/error_code.py | 6 +- .../errors/error_info.py | 44 ++++++------- .../server/auth/auth_middleware.py | 6 +- .../server/flight_rpc/flight_middleware.py | 8 +-- .../server/flight_rpc/flight_server.py | 18 ++--- .../server/flight_rpc/flight_service.py | 25 ++++--- .../server/flight_rpc/server_methods.py | 3 +- .../server/server_base.py | 8 +-- .../src/gooddata_flight_server/tasks/task.py | 6 +- .../tasks/task_error.py | 4 +- .../tasks/task_executor.py | 5 +- .../tasks/task_result.py | 22 +++---- .../tasks/temporal_container.py | 6 +- .../tasks/thread_task_executor.py | 20 +++--- .../gooddata_flight_server/utils/logging.py | 14 ++-- .../utils/methods_discovery.py | 3 +- .../utils/otel_tracing.py | 3 +- .../src/gooddata_pandas/data_access.py | 28 ++++---- .../src/gooddata_pandas/dataframe.py | 44 ++++++------- .../src/gooddata_pandas/good_pandas.py | 5 +- .../src/gooddata_pandas/result_convertor.py | 38 +++++------ .../src/gooddata_pandas/series.py | 18 ++--- .../src/gooddata_pandas/utils.py | 10 +-- .../dataframe/test_dataframe_for_exec_def.py | 8 +-- .../src/tests_support/vcrpy_utils.py | 6 +- pyproject.toml | 2 - 56 files changed, 354 insertions(+), 339 deletions(-) diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py index 972088bfd..cb0c90240 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/metrics.py @@ -1,7 +1,6 @@ # (C) 2023 GoodData Corporation import json import re -from typing import Optional import attrs from gooddata_sdk import CatalogDeclarativeMetric, CatalogDeclarativeModel @@ -26,8 +25,8 @@ @attrs.define(auto_attribs=True, kw_only=True) class DbtModelMetaGoodDataMetricProps(Base): - model_id: Optional[str] = None - format: Optional[str] = None + model_id: str | None = None + format: str | None = None @attrs.define(auto_attribs=True, kw_only=True) @@ -49,11 +48,11 @@ class DbtModelMetric(DbtModelBase): model: str calculation_method: str expression: str - filters: Optional[list[DbtModelMetricFilter]] = None + filters: list[DbtModelMetricFilter] | None = None class DbtModelMetrics: - def __init__(self, model_ids: Optional[list[str]], ldm: CatalogDeclarativeModel) -> None: + def __init__(self, model_ids: list[str] | None, ldm: CatalogDeclarativeModel) -> None: self.model_ids = model_ids self.ldm = ldm with open(DBT_PATH_TO_MANIFEST) as fp: @@ -104,7 +103,7 @@ def get_entity_type(self, table_name: str, expression_entity: str) -> str: else: raise Exception(f"Unsupported entity type {table_name=} {expression_entity=}") - def make_entity_id(self, table_name: str, token: str) -> Optional[str]: + def make_entity_id(self, table_name: str, token: str) -> str | None: entity_type = self.get_entity_type(table_name, token) if not entity_type: return None @@ -125,7 +124,7 @@ def resolve_entities_in_expression(self, expression: str, table_name: str) -> st result_tokens.append(entity_id or token) return " ".join(result_tokens) - def make_gooddata_filter(self, table_name: str, dbt_filters: Optional[list[DbtModelMetricFilter]] = None) -> str: + def make_gooddata_filter(self, table_name: str, dbt_filters: list[DbtModelMetricFilter] | None = None) -> str: # TODO - Quite naive implementation # e.g. missing polishing of values (e.g. SQL vs MAQL enclosers) gd_maql_filters = [] diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py index 8236840db..c50733ae0 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/profiles.py @@ -2,7 +2,7 @@ import argparse import os import re -from typing import Optional, Union +from typing import Union from urllib.parse import quote_plus import attrs @@ -97,7 +97,7 @@ class DbtOutputSnowflake(Base): database: str warehouse: str schema: str - query_tag: Optional[str] = None + query_tag: str | None = None def to_gooddata(self, data_source_id: str, schema_name: str) -> CatalogDataSourceSnowflake: return CatalogDataSourceSnowflake( @@ -222,7 +222,7 @@ def inject_env_vars(output_def: dict) -> None: # else do nothing, real value seems to be stored in dbt profile @staticmethod - def to_data_class(output: str, output_def: dict) -> Optional[DbtOutput]: + def to_data_class(output: str, output_def: dict) -> DbtOutput | None: db_type = output_def["type"] if db_type == "postgres": return DbtOutputPostgreSQL.from_dict({"name": output, **output_def}) diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py index 36aa971e2..36f1dcddf 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py @@ -3,7 +3,7 @@ import json import re from pathlib import Path -from typing import Optional, Union +from typing import Union import attrs from gooddata_sdk import CatalogDeclarativeColumn, CatalogDeclarativeTable, CatalogDeclarativeTables @@ -27,22 +27,22 @@ @attrs.define(auto_attribs=True, kw_only=True) class DbtModelMetaGoodDataTableProps(Base): - model_id: Optional[str] = None + model_id: str | None = None @attrs.define(auto_attribs=True, kw_only=True) class DbtModelMetaGoodDataColumnProps(Base): - id: Optional[str] = None - ldm_type: Optional[GoodDataLdmType] = None - referenced_table: Optional[str] = None - label_type: Optional[GoodDataLabelType] = None - attribute_column: Optional[str] = None - sort_column: Optional[str] = None - sort_direction: Optional[GoodDataSortDirection] = None - default_view: Optional[bool] = None + id: str | None = None + ldm_type: GoodDataLdmType | None = None + referenced_table: str | None = None + label_type: GoodDataLabelType | None = None + attribute_column: str | None = None + sort_column: str | None = None + sort_direction: GoodDataSortDirection | None = None + default_view: bool | None = None @property - def gooddata_ref_table_ldm_id(self) -> Optional[str]: + def gooddata_ref_table_ldm_id(self) -> str | None: if self.referenced_table: return self.referenced_table.lower() return None @@ -73,7 +73,7 @@ class DbtModelBase(Base): tags: list[str] # If 2+ references point to the same table, the table plays multiple roles, # it must be generated as multiple datasets - role_name: Optional[str] = None + role_name: str | None = None # TODO - duplicate of backend logic. # Solution: use result of generateLdm as a master template, and override based on dbt metadata only if necessary @@ -120,7 +120,7 @@ def upper_case_name(self) -> None: @attrs.define(auto_attribs=True, kw_only=True) class DbtModelColumn(DbtModelBase): - data_type: Optional[str] + data_type: str | None meta: DbtModelMetaGoodDataColumn = attrs.field(factory=DbtModelMetaGoodDataColumn) # Enable to override LDM ID for LDM entities derived from columns (attributes, ...) @@ -130,21 +130,21 @@ def ldm_id(self) -> str: return self.meta.gooddata.id or self.gooddata_ldm_id @property - def ldm_type(self) -> Optional[str]: + def ldm_type(self) -> str | None: if self.meta.gooddata.ldm_type is None: return None else: return self.meta.gooddata.ldm_type.value @property - def label_type(self) -> Optional[str]: + def label_type(self) -> str | None: if self.meta.gooddata.label_type is None: return None else: return self.meta.gooddata.label_type.value @property - def sort_direction(self) -> Optional[str]: + def sort_direction(self) -> str | None: if self.meta.gooddata.sort_direction is None: return None else: @@ -387,7 +387,7 @@ def make_facts(table: DbtModelTable) -> list[dict]: return facts @staticmethod - def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], Optional[dict]]: + def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple[list[dict], dict | None]: labels = [] default_view = None for column in table.columns.values(): @@ -507,7 +507,7 @@ def populate_role_playing_tables(tables: list[DbtModelTable], role_playing_table result.append(table) return result - def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[list[str]]) -> dict: + def make_declarative_datasets(self, data_source_id: str, model_ids: list[str] | None) -> dict: result: dict[str, list] = {"datasets": [], "date_instances": []} model_tables = [t for t in self.tables if not model_ids or t.meta.gooddata.model_id in model_ids] role_playing_tables = self.find_role_playing_tables(model_tables) @@ -517,7 +517,7 @@ def make_declarative_datasets(self, data_source_id: str, model_ids: Optional[lis result = self.make_dataset(data_source_id, table, role_playing_tables, result) return result - def get_entity_type(self, table_name: str, column_name: str) -> Optional[str]: + def get_entity_type(self, table_name: str, column_name: str) -> str | None: comp_table_name = table_name if self.upper_case: comp_table_name = table_name.upper() diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py index 5dfe103ab..6b67030b3 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py @@ -5,7 +5,6 @@ from asyncio import Semaphore from pathlib import Path from time import time -from typing import Optional import tabulate import yaml @@ -36,7 +35,7 @@ def generate_and_put_ldm( data_source_id: str, workspace_id: str, dbt_tables: DbtModelTables, - model_ids: Optional[list[str]], + model_ids: list[str] | None, ) -> None: scan_request = CatalogScanModelRequest(scan_tables=True, scan_views=True) logger.info(f"Scan data source {data_source_id=}") @@ -68,7 +67,7 @@ def deploy_ldm( args: Namespace, all_model_ids: list[str], sdk_wrapper: GoodDataSdkWrapper, - model_ids: Optional[list[str]], + model_ids: list[str] | None, workspace_id: str, ) -> None: logger.info("Generate and put LDM") @@ -186,7 +185,7 @@ async def test_visualizations( logger: logging.Logger, sdk_wrapper: GoodDataSdkWrapper, workspace_id: str, - skip_tests: Optional[list[str]], + skip_tests: list[str] | None, test_visualizations_parallelism: int = 1, ) -> None: start = time() @@ -334,7 +333,7 @@ def process_organization( logger: logging.Logger, sdk_wrapper: GoodDataSdkWrapper, gd_config: GoodDataConfig, - organization: Optional[GoodDataConfigOrganization] = None, + organization: GoodDataConfigOrganization | None = None, ) -> None: if args.method == "upload_notification": dbt_profiles = DbtProfiles(args) diff --git a/packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py b/packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py index 0e02a9d61..5e675ad78 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/gooddata/config.py @@ -1,5 +1,4 @@ # (C) 2023 GoodData Corporation -from typing import Optional import attr import attrs @@ -37,8 +36,8 @@ class GoodDataConfigProduct(Base): name: str environment_setup_id: str model_ids: list[str] = attr.field(factory=list) - localization: Optional[GoodDataConfigLocalization] = None - skip_tests: Optional[list[str]] = None + localization: GoodDataConfigLocalization | None = None + skip_tests: list[str] | None = None @attrs.define(auto_attribs=True, kw_only=True) @@ -49,7 +48,7 @@ class GoodDataConfigOrganization(Base): @attrs.define(auto_attribs=True, kw_only=True) class GoodDataGlobalConfig(Base): - test_visualizations_parallelism: Optional[int] = 1 + test_visualizations_parallelism: int | None = 1 @attrs.define(auto_attribs=True, kw_only=True) diff --git a/packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py b/packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py index bec10a40f..7b3db8b53 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/sdk_wrapper.py @@ -1,7 +1,6 @@ # (C) 2023 GoodData Corporation import argparse from logging import Logger -from typing import Optional from gooddata_sdk import GoodDataSdk @@ -11,7 +10,7 @@ class GoodDataSdkWrapper: # Timeout=600 because supporting waiting for GoodData services to start def __init__( - self, args: argparse.Namespace, logger: Logger, profile: Optional[str] = None, timeout: int = 600 + self, args: argparse.Namespace, logger: Logger, profile: str | None = None, timeout: int = 600 ) -> None: self.args = args self.logger = logger @@ -22,7 +21,7 @@ def __init__( if not self.args.dry_run: self.wait_for_gooddata_is_up(self.timeout) - def get_host_from_sdk(self) -> Optional[str]: + def get_host_from_sdk(self) -> str | None: # TODO - make _hostname public in gooddata_sdk return self.sdk.client._hostname @@ -54,7 +53,7 @@ def wait_for_gooddata_is_up(self, timeout: int) -> None: self.sdk.support.wait_till_available(timeout=timeout) self.logger.info(f"Host {host} is up") - def pre_cache_visualizations(self, workspaces: Optional[list] = None) -> None: + def pre_cache_visualizations(self, workspaces: list | None = None) -> None: if not workspaces: workspaces = [w.id for w in self.sdk.catalog_workspace.list_workspaces()] for workspace_id in workspaces: diff --git a/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json b/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json index 24e01d028..582366284 100644 --- a/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json +++ b/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json @@ -1860,6 +1860,20 @@ "data_type": null, "quote": null, "tags": [] + }, + "country": { + "name": "country", + "description": "", + "meta": { + "gooddata": { + "ldm_type": "label", + "label_type": "GEO_AREA", + "attribute_column": "code" + } + }, + "data_type": null, + "quote": null, + "tags": [] } }, "meta": { diff --git a/packages/gooddata-dbt/tests/resources/gooddata_layouts/pdm/airports.yaml b/packages/gooddata-dbt/tests/resources/gooddata_layouts/pdm/airports.yaml index 13ad88856..57dbecc35 100644 --- a/packages/gooddata-dbt/tests/resources/gooddata_layouts/pdm/airports.yaml +++ b/packages/gooddata-dbt/tests/resources/gooddata_layouts/pdm/airports.yaml @@ -21,6 +21,9 @@ columns: - dataType: STRING isPrimaryKey: false name: name + - dataType: STRING + isPrimaryKey: false + name: country - dataType: STRING isPrimaryKey: false name: state diff --git a/packages/gooddata-dbt/tests/test_tables.py b/packages/gooddata-dbt/tests/test_tables.py index 5b8eaafe5..fcfc74e62 100644 --- a/packages/gooddata-dbt/tests/test_tables.py +++ b/packages/gooddata-dbt/tests/test_tables.py @@ -49,3 +49,31 @@ def test_make_ldm(): assert ldm.ldm is not None assert len(ldm.ldm.datasets) == 4 assert len(ldm.ldm.date_instances) == 4 + + +FAA_MODEL_ID = "faa" + + +def test_make_ldm_geo_area(): + """Test that GEO_AREA label type is parsed and included in the LDM.""" + tables = DbtModelTables.from_local(upper_case=False, all_model_ids=[FAA_MODEL_ID], manifest_path=_MANIFEST_PATH) + scan_pdm = CatalogDeclarativeTables.load_from_disk(_PDM_PATH) + tables.set_data_types(scan_pdm) + data_source_id = "postgres" + + declarative_datasets = tables.make_declarative_datasets(data_source_id, [FAA_MODEL_ID]) + ldm = CatalogDeclarativeModel.from_dict({"ldm": declarative_datasets}, camel_case=False) + + assert ldm.ldm is not None + # airports is referenced twice (origin/destination), find either one + airports = [ds for ds in ldm.ldm.datasets if ds.id == "airports_origin"] + assert len(airports) == 1 + airports_ds = airports[0] + # Find the code attribute (which has geo labels) + code_attrs = [a for a in airports_ds.attributes if a.id == "code_origin"] + assert len(code_attrs) == 1 + labels = code_attrs[0].labels + label_types = {label.id: label.value_type for label in labels} + assert label_types["latitude_origin"] == "GEO_LATITUDE" + assert label_types["longitude_origin"] == "GEO_LONGITUDE" + assert label_types["country_origin"] == "GEO_AREA" diff --git a/packages/gooddata-fdw/src/gooddata_fdw/column_utils.py b/packages/gooddata-fdw/src/gooddata_fdw/column_utils.py index 095a6a184..2dbda909e 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/column_utils.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/column_utils.py @@ -1,7 +1,7 @@ # (C) 2022 GoodData Corporation from __future__ import annotations -from typing import Optional, Union +from typing import Union import gooddata_sdk as sdk from gooddata_sdk import Attribute, CatalogAttribute, Metric @@ -26,7 +26,7 @@ def table_col_as_computable(col: ColumnDefinition) -> Union[Attribute, Metric]: ) -def column_data_type_for(attribute: Optional[CatalogAttribute]) -> str: +def column_data_type_for(attribute: CatalogAttribute | None) -> str: """ Determine what postgres type should be used for `attribute`. diff --git a/packages/gooddata-fdw/src/gooddata_fdw/environment.py b/packages/gooddata-fdw/src/gooddata_fdw/environment.py index 8369ced64..33e216689 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/environment.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/environment.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Any, Optional, Union +from typing import Any, Union try: import multicorn @@ -88,12 +88,12 @@ def import_schema( schema: str, srv_options: dict[str, str], options: dict[str, str], - restriction_type: Optional[str], + restriction_type: str | None, restricts: list[str], ) -> list[TableDefinition]: return NotImplemented - def execute(self, quals: list[Qual], columns: list[str], sortkeys: Optional[list[Any]] = None): + def execute(self, quals: list[Qual], columns: list[str], sortkeys: list[Any] | None = None): pass ForeignDataWrapper = ForeignDataWrapperStub diff --git a/packages/gooddata-fdw/src/gooddata_fdw/executor.py b/packages/gooddata-fdw/src/gooddata_fdw/executor.py index 8e79a91e8..f884a377c 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/executor.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/executor.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Generator -from typing import Any, NamedTuple, Optional +from typing import Any, NamedTuple from gooddata_sdk import GoodDataSdk @@ -37,7 +37,7 @@ def validate_columns_def(self) -> None: validator.validate(column_name, column_def) def execute( - self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None + self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None ) -> Generator[dict[str, Any], None, None]: raise NotImplementedError() @@ -59,7 +59,7 @@ def can_react(cls, inputs: InitData) -> bool: return inputs.table_options.insight is not None def execute( - self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None + self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None ) -> Generator[dict[str, Any], None, None]: results_reader = InsightTableResultReader(self._table_columns, columns) insight = self._sdk.visualizations.get_visualization(self._workspace, self._insight) @@ -81,7 +81,7 @@ def can_react(cls, inputs: InitData) -> bool: return inputs.table_options.compute is not None def execute( - self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None + self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None ) -> Generator[dict[str, Any], None, None]: col_val.validate_columns_in_table_def(self._table_columns, columns) items = [column_utils.table_col_as_computable(self._table_columns[col_name]) for col_name in columns] @@ -105,7 +105,7 @@ def can_react(cls, inputs: InitData) -> bool: return True def execute( - self, quals: list[Qual], columns: list[str], sort_keys: Optional[list[Any]] = None + self, quals: list[Qual], columns: list[str], sort_keys: list[Any] | None = None ) -> Generator[dict[str, Any], None, None]: items = [column_utils.table_col_as_computable(col) for col in self._table_columns.values()] # TODO: pushdown more filters that are included in quals diff --git a/packages/gooddata-fdw/src/gooddata_fdw/fdw.py b/packages/gooddata-fdw/src/gooddata_fdw/fdw.py index 46596b532..d0ad831eb 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/fdw.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/fdw.py @@ -2,7 +2,7 @@ from __future__ import annotations import traceback -from typing import Any, Optional +from typing import Any from gooddata_sdk import GoodDataSdk @@ -34,7 +34,7 @@ def __init__(self, options: dict[str, str], columns: dict[str, ColumnDefinition] self._executor = ExecutorFactory.create(InitData(gd_sdk, self._server_options, self._table_options, columns)) self._executor.validate_columns_def() - def execute(self, quals: list[Qual], columns: list[str], sortkeys: Optional[list[Any]] = None): + def execute(self, quals: list[Qual], columns: list[str], sortkeys: list[Any] | None = None): _log_debug(f"query in fdw with {self._server_options}; {self._table_options}; columns {columns}; quals={quals}") try: return self._executor.execute(quals, columns, sortkeys) @@ -48,7 +48,7 @@ def import_schema( schema: str, srv_options: dict[str, str], options: dict[str, str], - restriction_type: Optional[str], + restriction_type: str | None, restricts: list[str], ) -> list[TableDefinition]: _log_info( diff --git a/packages/gooddata-fdw/src/gooddata_fdw/import_workspace.py b/packages/gooddata-fdw/src/gooddata_fdw/import_workspace.py index f0a2a4c56..3d32c7829 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/import_workspace.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/import_workspace.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from typing import NamedTuple, Optional +from typing import NamedTuple from gooddata_sdk import ( CatalogWorkspaceContent, @@ -24,7 +24,7 @@ from gooddata_fdw.pg_logging import _log_debug, _log_info, _log_warn -def _metric_format_to_precision(metric_format: Optional[str]) -> Optional[str]: +def _metric_format_to_precision(metric_format: str | None) -> str | None: if metric_format: re_decimal_places = re.compile(r"^[^.]+\.([#0]+)") match = re_decimal_places.search(metric_format) @@ -39,7 +39,7 @@ class ImporterInitData(NamedTuple): workspace: str server_options: options.ServerOptions import_options: options.ImportSchemaOptions - restriction_type: Optional[str] + restriction_type: str | None restricts: list[str] @@ -165,7 +165,7 @@ def _metric_to_table_column( # InsightMetric do not contain format in case of stored metrics @staticmethod - def _get_insight_metric_format(metric: VisualizationMetric, catalog: CatalogWorkspaceContent) -> Optional[str]: + def _get_insight_metric_format(metric: VisualizationMetric, catalog: CatalogWorkspaceContent) -> str | None: if metric.format: return metric.format elif metric.item_id: diff --git a/packages/gooddata-fdw/src/gooddata_fdw/naming.py b/packages/gooddata-fdw/src/gooddata_fdw/naming.py index eb5077a63..fc8cce043 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/naming.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/naming.py @@ -1,12 +1,10 @@ # (C) 2021 GoodData Corporation from __future__ import annotations -from typing import Optional - import gooddata_sdk as sdk -def _sanitize_str_for_postgres(string: str, used_names: Optional[dict[str, bool]] = None) -> str: +def _sanitize_str_for_postgres(string: str, used_names: dict[str, bool] | None = None) -> str: # replace non-alpha-num stuff with underscores with_underscores = "".join(char if char.isalnum() else "_" for char in string.lower()) diff --git a/packages/gooddata-fdw/src/gooddata_fdw/options.py b/packages/gooddata-fdw/src/gooddata_fdw/options.py index fbe81db7a..905657514 100644 --- a/packages/gooddata-fdw/src/gooddata_fdw/options.py +++ b/packages/gooddata-fdw/src/gooddata_fdw/options.py @@ -1,11 +1,11 @@ # (C) 2022 GoodData Corporation from __future__ import annotations -from typing import Any, Optional, Union +from typing import Any, Union class BaseOptions: - def __init__(self, validate: bool = True, skip_attributes: Optional[list[str]] = None) -> None: + def __init__(self, validate: bool = True, skip_attributes: list[str] | None = None) -> None: if skip_attributes is None: self._skip_attributes = [] else: @@ -124,6 +124,6 @@ def _validate_object_type(value: str) -> None: def numeric_max_size(self) -> str: return self._options.get("numeric_max_size", self.METRIC_DIGITS_BEFORE_DEC_POINT_DEFAULT) - def metric_data_type(self, precision: Optional[str] = None) -> str: + def metric_data_type(self, precision: str | None = None) -> str: digits_after = precision if precision else self.METRIC_DIGITS_AFTER_DEC_POINT_DEFAULT return f"DECIMAL({self.numeric_max_size}, {digits_after})" diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/data_source_messages.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/data_source_messages.py index a90186099..20aa3a361 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/data_source_messages.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/data_source_messages.py @@ -1,7 +1,7 @@ # (C) 2025 GoodData Corporation from collections.abc import Iterable from dataclasses import dataclass -from typing import Any, Optional +from typing import Any import orjson import pyarrow @@ -33,7 +33,7 @@ class DataSourceMessage: Type of the message, currently free-form, we might define some enum for these in the future. """ - data: Optional[Any] = None + data: Any | None = None """ Optional message-specific data. This can be anything that can be JSON-serialized. Try to keep this as small as possible: the backend has a quite strict size limit on the messages. @@ -47,7 +47,7 @@ def to_dict(self) -> dict[str, Any]: def add_data_source_messages_metadata( - data_source_messages: Iterable[DataSourceMessage], original_metadata: Optional[dict] = None + data_source_messages: Iterable[DataSourceMessage], original_metadata: dict | None = None ) -> dict[bytes, bytes]: """ Given a list of DataSourceMessages, creates a PyArrow-compatible metadata dictionary. diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/execution_context.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/execution_context.py index f988a38d4..b142cda16 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/execution_context.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/execution_context.py @@ -21,13 +21,13 @@ TResult = TypeVar("TResult") -def none_safe(func: Callable[[TInput], TResult]) -> Callable[[Optional[TInput]], Optional[TResult]]: +def none_safe(func: Callable[[TInput], TResult]) -> Callable[[TInput | None], TResult | None]: """ Decorator that makes the unary function safe for None input. If the only argument is None, the function returns None. """ - def wrapper(arg: Optional[TInput]) -> Optional[TResult]: + def wrapper(arg: TInput | None) -> TResult | None: if arg is None: return None return func(arg) @@ -114,18 +114,18 @@ class ExecutionContextAttribute: Title of the particular label used. """ - date_granularity: Optional[str] + date_granularity: str | None """ Date granularity of the attribute if it is a date attribute. """ - sorting: Optional[ExecutionContextAttributeSorting] + sorting: ExecutionContextAttributeSorting | None """ Sorting of the attribute. If not present, the attribute is not sorted. """ @staticmethod - def from_dict(d: Optional[dict]) -> Optional["ExecutionContextAttribute"]: + def from_dict(d: dict | None) -> Optional["ExecutionContextAttribute"]: """ Create ExecutionContextAttribute from a dictionary. :param d: the dictionary to parse @@ -153,7 +153,7 @@ class ExecutionContextPositiveAttributeFilter: Identifier of the label used. """ - values: list[Optional[str]] + values: list[str | None] """ Values of the filter. """ @@ -170,7 +170,7 @@ class ExecutionContextNegativeAttributeFilter: Identifier of the label used. """ - values: list[Optional[str]] + values: list[str | None] """ Values of the filter. """ @@ -366,17 +366,17 @@ class LabelElementsExecutionRequest: The label to get the elements for. """ - offset: Optional[int] + offset: int | None """ The number of elements to skip before returning. """ - limit: Optional[int] + limit: int | None """ The maximum number of elements to return. """ - exclude_primary_label: Optional[bool] + exclude_primary_label: bool | None """ Excludes items from the result that differ only by primary label @@ -384,33 +384,33 @@ class LabelElementsExecutionRequest: * true - return items with distinct requested label """ - exact_filter: Optional[list[str]] + exact_filter: list[str] | None """ Exact values to filter the elements by. """ - pattern_filter: Optional[str] + pattern_filter: str | None """ Filter the elements by a pattern. The pattern is matched against the element values in a case-insensitive way. """ - complement_filter: Optional[bool] + complement_filter: bool | None """ Whether to invert the effects of exact_filter amd pattern_filter. """ - depends_on: Optional[list[DependsOn]] + depends_on: list[DependsOn] | None """ Other labels or date filters that should be used to limit the elements. """ - filter_by: Optional[CatalogFilterBy] + filter_by: CatalogFilterBy | None """ Which label is used for filtering - primary or requested. If omitted the server will use the default value of "REQUESTED". """ - validate_by: Optional[list[CatalogValidateByItem]] + validate_by: list[CatalogValidateByItem] | None """ Other metrics, attributes, labels or facts used to validate the elements. """ @@ -442,17 +442,17 @@ class ExecutionInitiatorDisplay: Information about an execution being run in order to display the data in the UI. """ - dashboard_id: Optional[str] + dashboard_id: str | None """ The id of the dashboard the execution was run as a part of. """ - visualization_id: Optional[str] + visualization_id: str | None """ The id of the visualization the execution was run as a part of. """ - widget_id: Optional[str] + widget_id: str | None """ The id of the widget the execution was run as a part of. """ @@ -464,22 +464,22 @@ class ExecutionInitiatorAdHocExport: Information about an execution being run in order to export the data by a user in the UI. """ - dashboard_id: Optional[str] + dashboard_id: str | None """ The id of the dashboard the execution was run as a part of. """ - visualization_id: Optional[str] + visualization_id: str | None """ The id of the visualization the execution was run as a part of. """ - widget_id: Optional[str] + widget_id: str | None """ The id of the widget the execution was run as a part of. """ - export_type: Optional[str] + export_type: str | None """ The type of the exported file (CSV, RAW_CSV, etc.). """ @@ -491,7 +491,7 @@ class ExecutionInitiatorAutomation: Information about an execution being run because of an automation. """ - automation_id: Optional[str] + automation_id: str | None """ The id of the automation initiating this execution. """ @@ -503,17 +503,17 @@ class ExecutionInitiatorAlert: Information about an execution being run in order to evaluate an alert. """ - dashboard_id: Optional[str] + dashboard_id: str | None """ The id of the dashboard the execution was run as a part of. """ - visualization_id: Optional[str] + visualization_id: str | None """ The id of the visualization the execution was run as a part of. """ - widget_id: Optional[str] + widget_id: str | None """ The id of the widget the execution was run as a part of. """ @@ -624,18 +624,18 @@ class ExecutionContext: The ID of the user that invoked the FlexConnect function. """ - timestamp: Optional[str] + timestamp: str | None """ The timestamp of the execution used as "now" in date filters. For example 2020-06-03T10:15:30+01:00. """ - timezone: Optional[str] + timezone: str | None """ The timezone of the execution. """ - week_start: Optional[str] + week_start: str | None """ The start of the week. Either "monday" or "sunday". """ @@ -650,19 +650,19 @@ class ExecutionContext: All the attribute and date filters that are part of the execution request. """ - report_execution_request: Optional[ReportExecutionRequest] + report_execution_request: ReportExecutionRequest | None """ The report execution request that the FlexConnect function should process. Only present if the execution type is "REPORT". """ - label_elements_execution_request: Optional[LabelElementsExecutionRequest] + label_elements_execution_request: LabelElementsExecutionRequest | None """ The label elements execution request that the FlexConnect function should process. Only present if the execution type is "LABEL_ELEMENTS". """ - execution_initiator: Optional[ExecutionInitiator] + execution_initiator: ExecutionInitiator | None """ Information about what triggered this execution (e.g. display, export, automation, alert). """ diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/flight_methods.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/flight_methods.py index 863f992b3..deb3912d3 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/flight_methods.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/flight_methods.py @@ -1,7 +1,6 @@ # (C) 2024 GoodData Corporation import time from collections.abc import Generator -from typing import Optional import orjson import pyarrow.flight @@ -59,7 +58,7 @@ def __init__( self._poll_interval = poll_interval_ms / 1000 @staticmethod - def _create_descriptor(fun_name: str, metadata: Optional[dict]) -> pyarrow.flight.FlightDescriptor: + def _create_descriptor(fun_name: str, metadata: dict | None) -> pyarrow.flight.FlightDescriptor: cmd = { "functionName": fun_name, "metadata": metadata, @@ -97,9 +96,7 @@ def _prepare_task( cmd=submit_invocation.command, ) - def _prepare_flight_info( - self, task_id: str, task_result: Optional[TaskExecutionResult] - ) -> pyarrow.flight.FlightInfo: + def _prepare_flight_info(self, task_id: str, task_result: TaskExecutionResult | None) -> pyarrow.flight.FlightInfo: if task_result is None: raise ErrorInfo.for_reason( ErrorCode.BAD_ARGUMENT, f"Task with id '{task_id}' does not exist." @@ -142,7 +139,7 @@ def _get_flight_info_no_polling( structlog.contextvars.bind_contextvars(peer=context.peer()) invocation = extract_submit_invocation_from_descriptor(descriptor) - task: Optional[FlexConnectFunctionTask] = None + task: FlexConnectFunctionTask | None = None try: task = self._prepare_task(context, invocation) @@ -192,7 +189,7 @@ def _get_flight_info_polling( invocation = extract_pollable_invocation_from_descriptor(descriptor) task_id: str - fun_name: Optional[str] = None + fun_name: str | None = None if isinstance(invocation, CancelInvocation): # cancel the given task and raise cancellation exception diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function.py index 8929d57fd..31f5043b8 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function.py @@ -1,6 +1,5 @@ # (C) 2024 GoodData Corporation import abc -from typing import Optional import pyarrow from gooddata_flight_server import ArrowData, ServerContext @@ -37,17 +36,17 @@ class FlexConnectFunction(abc.ABC): for every call using the `create` method. """ - Name: Optional[str] = None + Name: str | None = None """ Function MUST define a unique name """ - Schema: Optional[pyarrow.Schema] = None + Schema: pyarrow.Schema | None = None """ Function MUST define schema describing its data. """ - Metadata: Optional[dict] = None + Metadata: dict | None = None """ Function MAY provide additional metadata about themselves. These then influence how the function is used by and called from GoodData Cloud & FlexQuery. @@ -79,7 +78,7 @@ def create(cls) -> "FlexConnectFunction": def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: """ diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_invocation.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_invocation.py index 8647bf0ac..2a16f6590 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_invocation.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_invocation.py @@ -1,6 +1,6 @@ # (C) 2025 GoodData Corporation from dataclasses import dataclass -from typing import Optional, Union +from typing import Union import orjson import pyarrow.flight @@ -46,7 +46,7 @@ class SubmitInvocation: Parameters to pass to the FlexConnect function. """ - columns: Optional[tuple[str, ...]] + columns: tuple[str, ...] | None """ Columns to get from the FlexConnect function result. This may be used for column trimming by the function: the function must return at least those columns. diff --git a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_task.py b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_task.py index 0da016a3a..06a00f6df 100644 --- a/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_task.py +++ b/packages/gooddata-flexconnect/src/gooddata_flexconnect/function/function_task.py @@ -1,5 +1,5 @@ # (C) 2024 GoodData Corporation -from typing import Optional, Union +from typing import Union import structlog from gooddata_flight_server import FlightDataTaskResult, Task, TaskError, TaskResult @@ -16,11 +16,11 @@ def __init__( self, fun: FlexConnectFunction, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], cmd: bytes, cancellable: bool = True, - task_id: Optional[str] = None, + task_id: str | None = None, ): super().__init__(cmd, cancellable, task_id) @@ -32,7 +32,7 @@ def __init__( _LOGGER.info("flexconnect_task_created", fun=fun.Name, task_id=self._task_id) @property - def fun_name(self) -> Optional[str]: + def fun_name(self) -> str | None: return self._fun.Name def run(self) -> Union[TaskResult, TaskError]: diff --git a/packages/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py b/packages/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py index 3b9bb8f17..16a58dfe5 100644 --- a/packages/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py +++ b/packages/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py @@ -1,6 +1,5 @@ # (C) 2024 GoodData Corporation import pytest - from gooddata_flexconnect.function.execution_context import ( ExecutionContext, ExecutionContextAttribute, diff --git a/packages/gooddata-flexconnect/tests/function/testing_funs.py b/packages/gooddata-flexconnect/tests/function/testing_funs.py index f9e03aa3b..cc62aa581 100644 --- a/packages/gooddata-flexconnect/tests/function/testing_funs.py +++ b/packages/gooddata-flexconnect/tests/function/testing_funs.py @@ -1,5 +1,4 @@ # (C) 2024 GoodData Corporation -from typing import Optional import pyarrow from gooddata_flexconnect.function.function import FlexConnectFunction @@ -13,7 +12,7 @@ class Fun1(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: pass @@ -28,7 +27,7 @@ class Fun2(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: pass diff --git a/packages/gooddata-flexconnect/tests/server/funs/fun1.py b/packages/gooddata-flexconnect/tests/server/funs/fun1.py index c2518b464..be2c5db17 100644 --- a/packages/gooddata-flexconnect/tests/server/funs/fun1.py +++ b/packages/gooddata-flexconnect/tests/server/funs/fun1.py @@ -1,5 +1,4 @@ # (C) 2024 GoodData Corporation -from typing import Optional import pyarrow from gooddata_flexconnect.function.function import FlexConnectFunction @@ -19,7 +18,7 @@ class _SimpleFun1(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: return pyarrow.table( diff --git a/packages/gooddata-flexconnect/tests/server/funs/fun2.py b/packages/gooddata-flexconnect/tests/server/funs/fun2.py index b4d8b3864..4d91cc496 100644 --- a/packages/gooddata-flexconnect/tests/server/funs/fun2.py +++ b/packages/gooddata-flexconnect/tests/server/funs/fun2.py @@ -1,11 +1,10 @@ # (C) 2024 GoodData Corporation -from typing import Optional import pyarrow from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData, ServerContext -_DATA: Optional[pyarrow.Table] = None +_DATA: pyarrow.Table | None = None class _SimpleFun2(FlexConnectFunction): @@ -21,7 +20,7 @@ class _SimpleFun2(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: assert _DATA is not None diff --git a/packages/gooddata-flexconnect/tests/server/funs/fun3.py b/packages/gooddata-flexconnect/tests/server/funs/fun3.py index c5634c8c3..fbe5c62b7 100644 --- a/packages/gooddata-flexconnect/tests/server/funs/fun3.py +++ b/packages/gooddata-flexconnect/tests/server/funs/fun3.py @@ -1,12 +1,11 @@ # (C) 2024 GoodData Corporation import time -from typing import Optional import pyarrow from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData -_DATA: Optional[pyarrow.Table] = None +_DATA: pyarrow.Table | None = None class _LongRunningFun(FlexConnectFunction): @@ -22,7 +21,7 @@ class _LongRunningFun(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: # sleep is intentionally setup to be longer than the deadline for diff --git a/packages/gooddata-flexconnect/tests/server/funs/fun4.py b/packages/gooddata-flexconnect/tests/server/funs/fun4.py index 6e32b7298..049ae3c1e 100644 --- a/packages/gooddata-flexconnect/tests/server/funs/fun4.py +++ b/packages/gooddata-flexconnect/tests/server/funs/fun4.py @@ -1,12 +1,11 @@ # (C) 2024 GoodData Corporation import time -from typing import Optional import pyarrow from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData -_DATA: Optional[pyarrow.Table] = None +_DATA: pyarrow.Table | None = None class _PollableFun(FlexConnectFunction): @@ -22,7 +21,7 @@ class _PollableFun(FlexConnectFunction): def call( self, parameters: dict, - columns: Optional[tuple[str, ...]], + columns: tuple[str, ...] | None, headers: dict[str, list[str]], ) -> ArrowData: # sleep is intentionally setup to be longer than one polling interval diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/cli.py b/packages/gooddata-flight-server/src/gooddata_flight_server/cli.py index ebb0f6ecb..c11a50a83 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/cli.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/cli.py @@ -2,7 +2,7 @@ import argparse import sys import traceback -from typing import Optional, TypeVar +from typing import TypeVar from dynaconf import ValidationError @@ -85,7 +85,7 @@ def _create_server(args: argparse.Namespace) -> GoodDataFlightServer: # not really needed to be global, keeping it here so that instance of server is reachable # easily from the debugger -_SERVER: Optional[GoodDataFlightServer] = None +_SERVER: GoodDataFlightServer | None = None def server_cli() -> None: diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/config/config.py b/packages/gooddata-flight-server/src/gooddata_flight_server/config/config.py index 9b06a7ae2..1f8a3013a 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/config/config.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/config/config.py @@ -5,7 +5,7 @@ import platform import socket from dataclasses import dataclass -from typing import Any, Optional +from typing import Any from dynaconf import Dynaconf, ValidationError, Validator @@ -34,10 +34,10 @@ class AuthenticationMethod(enum.Enum): @dataclass(frozen=True) class OtelConfig: - exporter_type: Optional[OtelExporterType] + exporter_type: OtelExporterType | None service_name: str - service_namespace: Optional[str] - service_instance_id: Optional[str] + service_namespace: str | None + service_instance_id: str | None extract_context_from_headers: bool @@ -50,21 +50,21 @@ class ServerConfig: use_tls: bool use_mutual_tls: bool - tls_cert_and_key: Optional[tuple[bytes, bytes]] - tls_root_cert: Optional[bytes] + tls_cert_and_key: tuple[bytes, bytes] | None + tls_root_cert: bytes | None authentication_method: AuthenticationMethod - token_header_name: Optional[str] - token_verification: Optional[str] + token_header_name: str | None + token_verification: str | None task_threads: int task_close_threads: int task_result_ttl_sec: int - metrics_host: Optional[str] + metrics_host: str | None metrics_port: int - health_check_host: Optional[str] + health_check_host: str | None health_check_port: int malloc_trim_interval_sec: int @@ -77,8 +77,8 @@ def without_tls(self) -> "ServerConfig": def _basic_sanity(val: bytes) -> bytes: return val[0:38] + b"..." + val[-38:] - sanitized_root_cert: Optional[bytes] = None - sanitized_cert_and_key: Optional[tuple[bytes, bytes]] = None + sanitized_root_cert: bytes | None = None + sanitized_cert_and_key: tuple[bytes, bytes] | None = None if self.tls_root_cert is not None: sanitized_root_cert = _basic_sanity(self.tls_root_cert) @@ -422,7 +422,7 @@ def _validate_boolean(val: Any) -> bool: ] -def _read_tls_setting(settings: Dynaconf, setting: str) -> Optional[bytes]: +def _read_tls_setting(settings: Dynaconf, setting: str) -> bytes | None: value: str = settings.get(setting) if value is None: return None @@ -448,8 +448,8 @@ def _create_server_config(settings: Dynaconf) -> ServerConfig: advertise_port = server_settings.get(_Settings.AdvertisePort) or server_settings.get(_Settings.ListenPort) use_tls = server_settings.get(_Settings.UseTls) - tls_cert_and_key: Optional[tuple[bytes, bytes]] = None - tls_root_cert: Optional[bytes] = None + tls_cert_and_key: tuple[bytes, bytes] | None = None + tls_root_cert: bytes | None = None if use_tls: cert = _read_tls_setting(server_settings, _Settings.TlsCertificate) @@ -471,7 +471,7 @@ def _create_server_config(settings: Dynaconf) -> ServerConfig: tls_root_cert = _read_tls_setting(server_settings, _Settings.TlsRoot) _auth_method = AuthenticationMethod(server_settings.get(_Settings.AuthenticationMethod)) - _token_verification: Optional[str] = None + _token_verification: str | None = None if _auth_method == AuthenticationMethod.Token: _token_verification = server_settings.get(_Settings.TokenVerification) or _DEFAULT_TOKEN_VERIFICATION diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_code.py b/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_code.py index a8b6ea3cf..c78edbda3 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_code.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_code.py @@ -1,5 +1,5 @@ # (C) 2024 GoodData Corporation -from typing import Any, Literal, Optional +from typing import Any, Literal from typing_extensions import TypeAlias @@ -36,13 +36,13 @@ def _init_names(cls: Any) -> Any: } -def _get_flags(retry: Optional[_RetryFlagsLiteral] = None) -> int: +def _get_flags(retry: _RetryFlagsLiteral | None = None) -> int: if retry is None: return 0x0 return _FlagsMapping.get(retry, 0x0) -def _error_code(code: int, retry: Optional[_RetryFlagsLiteral] = None) -> int: +def _error_code(code: int, retry: _RetryFlagsLiteral | None = None) -> int: return _get_flags(retry) | code diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_info.py b/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_info.py index 998e95baa..65325fd4c 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_info.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/errors/error_info.py @@ -12,7 +12,7 @@ _ERROR_INFO_MAX_DETAIL = 512 -def _truncate_str_value(val: Optional[str], max_len: int) -> Optional[str]: +def _truncate_str_value(val: str | None, max_len: int) -> str | None: if val is None: return None @@ -34,13 +34,13 @@ class ErrorInfo: def __init__( self, msg: str, - detail: Optional[str] = None, - body: Optional[bytes] = None, + detail: str | None = None, + body: bytes | None = None, code: int = 0, ) -> None: self._msg = cast(str, _truncate_str_value(msg, _ERROR_INFO_MAX_MSG)) - self._detail: Optional[str] = _truncate_str_value(detail, _ERROR_INFO_MAX_DETAIL) - self._body: Optional[bytes] = body + self._detail: str | None = _truncate_str_value(detail, _ERROR_INFO_MAX_DETAIL) + self._body: bytes | None = body self._code: int = code @property @@ -58,14 +58,14 @@ def code(self) -> int: return self._code @property - def detail(self) -> Optional[str]: + def detail(self) -> str | None: """ :return: a human-readable error detail; if included may help with error diagnostics """ return self._detail @property - def body(self) -> Optional[bytes]: + def body(self) -> bytes | None: """ :return: error body, suitable for programmatic consumption; used by server to send structured information which the client code may want to work with @@ -83,7 +83,7 @@ def with_msg(self, msg: str) -> "ErrorInfo": return self - def with_detail(self, detail: Optional[str] = None) -> "ErrorInfo": + def with_detail(self, detail: str | None = None) -> "ErrorInfo": """ Updates or resets the error detail. @@ -95,7 +95,7 @@ def with_detail(self, detail: Optional[str] = None) -> "ErrorInfo": return self - def with_body(self, body: Optional[Union[bytes, str]]) -> "ErrorInfo": + def with_body(self, body: Union[bytes, str] | None) -> "ErrorInfo": """ Updates or resets the error body. @@ -200,7 +200,7 @@ def to_unauthorized_error(self) -> pyarrow.flight.FlightUnauthorizedError: def to_flight_error( self, - error_factory: Callable[[str, Optional[bytes]], pyarrow.flight.FlightError], + error_factory: Callable[[str, bytes | None], pyarrow.flight.FlightError], ) -> pyarrow.flight.FlightError: """ Uses the provided error factory - which can be for example the FlightError subclass, to create an @@ -221,7 +221,7 @@ def from_bytes(val: bytes) -> "ErrorInfo": """ try: _json = orjson.loads(val) - body: Optional[bytes] = base64.b64decode(_json["body"]) if _json.get("body") is not None else None + body: bytes | None = base64.b64decode(_json["body"]) if _json.get("body") is not None else None return ErrorInfo( msg=_json.get("msg"), @@ -283,7 +283,7 @@ def maybe_from_pyarrow_error( def for_exc( code: int, e: BaseException, - extra_msg: Optional[str] = None, + extra_msg: str | None = None, include_traceback: bool = True, ) -> "ErrorInfo": """ @@ -314,7 +314,7 @@ def for_exc( msg = f"{extra_msg}: {msg}" if include_traceback: - detail: Optional[str] = "".join(traceback.format_exception(None, e, e.__traceback__)) + detail: str | None = "".join(traceback.format_exception(None, e, e.__traceback__)) else: detail = None @@ -362,9 +362,9 @@ def bad_argument(msg: str) -> pyarrow.flight.FlightError: @staticmethod def poll( - flight_info: Optional[pyarrow.flight.FlightInfo] = None, - retry_descriptor: Optional[pyarrow.flight.FlightDescriptor] = None, - cancel_descriptor: Optional[pyarrow.flight.FlightDescriptor] = None, + flight_info: pyarrow.flight.FlightInfo | None = None, + retry_descriptor: pyarrow.flight.FlightDescriptor | None = None, + cancel_descriptor: pyarrow.flight.FlightDescriptor | None = None, ) -> pyarrow.flight.FlightTimedOutError: """ Convenience factory that creates FlightTimedOut error with POLL error code and `RetryInfo` which @@ -402,16 +402,16 @@ class RetryInfo: def __init__( self, - flight_info: Optional[pyarrow.flight.FlightInfo] = None, - retry_descriptor: Optional[pyarrow.flight.FlightDescriptor] = None, - cancel_descriptor: Optional[pyarrow.flight.FlightDescriptor] = None, + flight_info: pyarrow.flight.FlightInfo | None = None, + retry_descriptor: pyarrow.flight.FlightDescriptor | None = None, + cancel_descriptor: pyarrow.flight.FlightDescriptor | None = None, ) -> None: self._flight_info = flight_info self._retry_descriptor = retry_descriptor self._cancel_descriptor = cancel_descriptor @property - def flight_info(self) -> Optional[pyarrow.flight.FlightInfo]: + def flight_info(self) -> pyarrow.flight.FlightInfo | None: """ FlightInfo available at the time of the poll timeout. The information may be incomplete. The full FlightInfo is built in cumulative fashion - the subsequent @@ -425,7 +425,7 @@ def flight_info(self) -> Optional[pyarrow.flight.FlightInfo]: return self._flight_info @property - def retry_descriptor(self) -> Optional[pyarrow.flight.FlightDescriptor]: + def retry_descriptor(self) -> pyarrow.flight.FlightDescriptor | None: """ Returns descriptor that the client should use to retry the GetFlightInfo call in order to see whether the command has completed. @@ -437,7 +437,7 @@ def retry_descriptor(self) -> Optional[pyarrow.flight.FlightDescriptor]: return self._retry_descriptor @property - def cancel_descriptor(self) -> Optional[pyarrow.flight.FlightDescriptor]: + def cancel_descriptor(self) -> pyarrow.flight.FlightDescriptor | None: """ Returns descriptor that the client can use to cancel the command that is in progress. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/auth/auth_middleware.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/auth/auth_middleware.py index 1887f8eb1..b386d9e99 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/auth/auth_middleware.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/auth/auth_middleware.py @@ -1,5 +1,5 @@ # (C) 2024 GoodData Corporation -from typing import Any, Optional +from typing import Any import pyarrow.flight import structlog @@ -31,7 +31,7 @@ def token_data(self) -> Any: class TokenAuthMiddlewareFactory(pyarrow.flight.ServerMiddlewareFactory): def __init__( self, - token_header_name: Optional[str], + token_header_name: str | None, strategy: TokenVerificationStrategy, ): super().__init__() @@ -69,7 +69,7 @@ def _auth_header_value(lookup: str) -> str: token = _auth_header_value(self._token_header_name) return token.strip() - def start_call(self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]]) -> Optional[TokenAuthMiddleware]: + def start_call(self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]]) -> TokenAuthMiddleware | None: try: token = self._extract_token(headers) result = self._strategy.verify(call_info=info, token=token) diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_middleware.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_middleware.py index fa4a983f6..9874c9b2b 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_middleware.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_middleware.py @@ -1,5 +1,5 @@ # (C) 2024 GoodData Corporation -from typing import Any, Callable, Optional +from typing import Any, Callable import opentelemetry.context as otelctx import opentelemetry.propagate as otelpropagate @@ -46,7 +46,7 @@ def headers(self) -> dict[str, list[str]]: return self._headers -OnEndCallbackFn: TypeAlias = Callable[[Optional[pyarrow.ArrowException]], Any] +OnEndCallbackFn: TypeAlias = Callable[[pyarrow.ArrowException | None], Any] class CallFinalizer(pyarrow.flight.ServerMiddleware): @@ -85,7 +85,7 @@ def register_on_end(self, fun: OnEndCallbackFn) -> None: """ self._on_end.append(fun) - def call_completed(self, exception: Optional[pyarrow.lib.ArrowException]) -> None: + def call_completed(self, exception: pyarrow.lib.ArrowException | None) -> None: try: for fun in self._on_end: fun(exception) @@ -132,7 +132,7 @@ def call_tracing(self) -> tuple[otelctx.Context, trace.Span]: """ return self._otel_ctx, self._otel_span - def call_completed(self, exception: Optional[pyarrow.lib.ArrowException]) -> None: + def call_completed(self, exception: pyarrow.lib.ArrowException | None) -> None: # OpenTelemetry context/span restore; # # this has to happen because this method is done from thread managed diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_server.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_server.py index 6e116d68c..ba974668a 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_server.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_server.py @@ -10,7 +10,7 @@ """ from collections.abc import Generator -from typing import Any, Callable, Optional, TypeVar, Union +from typing import Any, Callable, TypeVar, Union import opentelemetry.context as otelctx import opentelemetry.trace as oteltrace @@ -22,7 +22,7 @@ FlightServerMethods, ) -FlightServerLocation: TypeAlias = Union[str, bytes, Optional[tuple[str, int]], pyarrow.flight.Location] +FlightServerLocation: TypeAlias = Union[str, bytes, tuple[str, int] | None, pyarrow.flight.Location] FlightTlsCertificates: TypeAlias = list[tuple[bytes, bytes]] FlightMiddlewares: TypeAlias = dict[str, pyarrow.flight.ServerMiddlewareFactory] @@ -82,13 +82,13 @@ class FlightServer(pyarrow.flight.FlightServerBase): def __init__( self, - methods: Optional[FlightServerMethods] = None, - location: Optional[FlightServerLocation] = None, - auth_handler: Optional[pyarrow.flight.ServerAuthHandler] = None, - tls_certificates: Optional[FlightTlsCertificates] = None, - verify_client: Optional[bool] = None, - root_certificates: Optional[bytes] = None, - middleware: Optional[FlightMiddlewares] = None, + methods: FlightServerMethods | None = None, + location: FlightServerLocation | None = None, + auth_handler: pyarrow.flight.ServerAuthHandler | None = None, + tls_certificates: FlightTlsCertificates | None = None, + verify_client: bool | None = None, + root_certificates: bytes | None = None, + middleware: FlightMiddlewares | None = None, ): """ Create Flight server diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_service.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_service.py index a0c10fb67..75de7b57b 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_service.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/flight_service.py @@ -3,7 +3,6 @@ # mypy: no-strict-optional from threading import Thread -from typing import Optional import pyarrow.flight import structlog @@ -41,14 +40,14 @@ class _AvailabilityMiddlewareFactory(pyarrow.flight.ServerMiddlewareFactory): If unavailable_reason is set -> reject. Otherwise, let the request through. """ - def __init__(self, unavailable_reason: Optional[ErrorInfo] = None): + def __init__(self, unavailable_reason: ErrorInfo | None = None): super().__init__() - self.unavailable_reason: Optional[ErrorInfo] = unavailable_reason + self.unavailable_reason: ErrorInfo | None = unavailable_reason def start_call( self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]] - ) -> Optional[pyarrow.flight.ServerMiddleware]: + ) -> pyarrow.flight.ServerMiddleware | None: if self.unavailable_reason is None: return None @@ -58,14 +57,14 @@ def start_call( class _CallInfoMiddlewareFactory(pyarrow.flight.ServerMiddlewareFactory): def start_call( self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]] - ) -> Optional[pyarrow.flight.ServerMiddleware]: + ) -> pyarrow.flight.ServerMiddleware | None: return CallInfo(info, headers) class _CallFinalizerMiddlewareFactory(pyarrow.flight.ServerMiddlewareFactory): def start_call( self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]] - ) -> Optional[pyarrow.flight.ServerMiddleware]: + ) -> pyarrow.flight.ServerMiddleware | None: return CallFinalizer() @@ -76,7 +75,7 @@ def __init__(self, extract_context: bool) -> None: def start_call( self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]] - ) -> Optional[pyarrow.flight.ServerMiddleware]: + ) -> pyarrow.flight.ServerMiddleware | None: return OtelMiddleware(info, headers, self._extract_context) @@ -110,13 +109,13 @@ def __init__( # internal mutable state # server starts immediately when constructed (PyArrow stuff); thus defer # construction until start() is called - self._server: Optional[FlightServer] = None - self._flight_shutdown_thread: Optional[Thread] = None + self._server: FlightServer | None = None + self._flight_shutdown_thread: Thread | None = None self._stopped = False def _initialize_authentication( self, ctx: ServerContext - ) -> Optional[tuple[str, pyarrow.flight.ServerMiddlewareFactory]]: + ) -> tuple[str, pyarrow.flight.ServerMiddlewareFactory] | None: if self._config.authentication_method == AuthenticationMethod.NoAuth: if self._config.use_mutual_tls: return None @@ -139,9 +138,7 @@ def _initialize_authentication( ctx.config.token_header_name, verification ) - def _initialize_otel_tracing( - self, ctx: ServerContext - ) -> Optional[tuple[str, pyarrow.flight.ServerMiddlewareFactory]]: + def _initialize_otel_tracing(self, ctx: ServerContext) -> tuple[str, pyarrow.flight.ServerMiddlewareFactory] | None: if self._config.otel_config.exporter_type is None: return None @@ -262,7 +259,7 @@ def _shutdown_server(self) -> None: self._server.shutdown() self._logger.info("flight_service_finished") - def wait_for_stop(self, timeout: Optional[float] = None) -> bool: + def wait_for_stop(self, timeout: float | None = None) -> bool: if self._flight_shutdown_thread is None: # this is really some mess in the caller code.. did not call stop() but tries to wait for it.. raise AssertionError("Flight server stop() was not issued yet attempting to wait for the server to stop.") diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/server_methods.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/server_methods.py index b83001506..958979732 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/server_methods.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/flight_rpc/server_methods.py @@ -1,6 +1,5 @@ # (C) 2024 GoodData Corporation from collections.abc import Generator -from typing import Optional import pyarrow.flight import structlog @@ -104,7 +103,7 @@ def do_get_task_result( rlock, data = result.acquire_data() - def _on_end(_: Optional[pyarrow.ArrowException]) -> None: + def _on_end(_: pyarrow.ArrowException | None) -> None: """ Once the request that streams the data out is done, make sure to release the read-lock. Single-use results are closed at diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/server/server_base.py b/packages/gooddata-flight-server/src/gooddata_flight_server/server/server_base.py index f9d24ccf3..ecd3067a1 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/server/server_base.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/server/server_base.py @@ -5,7 +5,7 @@ import signal from abc import abstractmethod from threading import Condition, Thread -from typing import Any, Optional +from typing import Any import pyarrow import structlog @@ -61,7 +61,7 @@ def __init__( # main server notifies on this condition once all sub-services are started self._start_cond = Condition() self._started = False - self._startup_interrupted: Optional[Exception] = None + self._startup_interrupted: Exception | None = None self._stop = False self._abort = False @@ -233,7 +233,7 @@ def aborted(self) -> bool: """ return self._abort - def wait_for_start(self, timeout: Optional[float] = None) -> bool: + def wait_for_start(self, timeout: float | None = None) -> bool: """ Waits until server and all its services are up and running. @@ -253,7 +253,7 @@ def wait_for_start(self, timeout: Optional[float] = None) -> bool: return True - def wait_for_stop(self, timeout: Optional[float] = None) -> bool: + def wait_for_stop(self, timeout: float | None = None) -> bool: """ Waits until the main server thread stops. If the server startup encountered error (and never started), then that error will be raised. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task.py index 86f384213..d432fbe45 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task.py @@ -3,7 +3,7 @@ import threading import uuid from concurrent.futures import CancelledError -from typing import Optional, Union, final +from typing import Union, final from gooddata_flight_server.tasks.task_error import TaskError from gooddata_flight_server.tasks.task_result import TaskResult @@ -46,7 +46,7 @@ def __init__( self, cmd: bytes, cancellable: bool = True, - task_id: Optional[str] = None, + task_id: str | None = None, ): self._task_id = task_id or uuid.uuid4().hex self._cmd = cmd @@ -142,7 +142,7 @@ def on_task_cancel(self) -> None: """ return - def on_task_error(self, error: TaskError) -> Optional[TaskError]: + def on_task_error(self, error: TaskError) -> TaskError | None: """ This method will be called when a task fails with and raises an exception. It will be called after executor creates an instance of TaskError from the diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_error.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_error.py index 9573e0f5e..e20ee3e5a 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_error.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_error.py @@ -1,7 +1,7 @@ # (C) 2024 GoodData Corporation import dataclasses from dataclasses import dataclass -from typing import Callable, Optional +from typing import Callable import pyarrow.flight @@ -36,7 +36,7 @@ class TaskError: """ error_info: ErrorInfo - error_factory: Callable[[str, Optional[bytes]], pyarrow.flight.FlightError] + error_factory: Callable[[str, bytes | None], pyarrow.flight.FlightError] client_error: bool = False """ diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_executor.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_executor.py index 58ef99fd1..53970f537 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_executor.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_executor.py @@ -1,6 +1,5 @@ # (C) 2024 GoodData Corporation import abc -from typing import Optional from gooddata_flight_server.tasks.task import Task from gooddata_flight_server.tasks.task_result import TaskExecutionResult @@ -55,7 +54,7 @@ def submit( raise NotImplementedError @abc.abstractmethod - def get_task_submitted_timestamp(self, task_id: str) -> Optional[float]: + def get_task_submitted_timestamp(self, task_id: str) -> float | None: """ Returns the timestamp of when the task with the given id was submitted. :param task_id: task id to get the timestamp for @@ -64,7 +63,7 @@ def get_task_submitted_timestamp(self, task_id: str) -> Optional[float]: raise NotImplementedError @abc.abstractmethod - def wait_for_result(self, task_id: str, timeout: Optional[float] = None) -> Optional[TaskExecutionResult]: + def wait_for_result(self, task_id: str, timeout: float | None = None) -> TaskExecutionResult | None: """ Wait for the task with the provided task id to finish. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_result.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_result.py index a0772bd4d..e5ecf4ab2 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_result.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/task_result.py @@ -3,7 +3,7 @@ import threading from collections.abc import Generator, Iterable from dataclasses import dataclass -from typing import Callable, Optional, Union, final +from typing import Callable, Union, final import pyarrow.flight import structlog @@ -49,7 +49,7 @@ def __init__(self, single_use_data: bool = False) -> None: self._claimed = False self._closed = False - def _acquire_reader(self) -> Optional[rwlock.Lockable]: + def _acquire_reader(self) -> rwlock.Lockable | None: rlock = self._data_lock.gen_rlock() if not rlock.acquire(blocking=False): # lock cannot be acquired -> means write lock is taken -> means data is being closed @@ -181,7 +181,7 @@ def close(self) -> None: self._close() @staticmethod - def for_table(table: pyarrow.Table, on_close: Optional[OnCloseCallback] = None) -> "FlightDataTaskResult": + def for_table(table: pyarrow.Table, on_close: OnCloseCallback | None = None) -> "FlightDataTaskResult": """ Factory to create result for an Arrow table. This result allows for repeated reads. @@ -196,7 +196,7 @@ def for_table(table: pyarrow.Table, on_close: Optional[OnCloseCallback] = None) @staticmethod def for_reader( - reader: pyarrow.RecordBatchReader, on_close: Optional[OnCloseCallback] = None + reader: pyarrow.RecordBatchReader, on_close: OnCloseCallback | None = None ) -> "FlightDataTaskResult": """ Factory to create result for an RecordBatchReader. The created result will @@ -212,7 +212,7 @@ def for_reader( return _ReaderTaskResult(reader, on_close=on_close) @staticmethod - def for_data(data: ArrowData, on_close: Optional[OnCloseCallback] = None) -> "FlightDataTaskResult": + def for_data(data: ArrowData, on_close: OnCloseCallback | None = None) -> "FlightDataTaskResult": """ Convenience factory function to create result from either Arrow Table or RecordBatchReader. @@ -262,9 +262,9 @@ def __init__( self, task_id: str, cmd: bytes, - result: Optional[TaskResult], + result: TaskResult | None, cancelled: bool, - error: Optional[TaskError], + error: TaskError | None, ): self._task_id = task_id self._cmd = cmd @@ -288,7 +288,7 @@ def cmd(self) -> bytes: return self._cmd @property - def result(self) -> Optional[TaskResult]: + def result(self) -> TaskResult | None: """ :return: result of task's successful execution; None if the task failed or was cancelled """ @@ -302,7 +302,7 @@ def cancelled(self) -> bool: return self._cancelled @property - def error(self) -> Optional[TaskError]: + def error(self) -> TaskError | None: """ :return: error that caused the task to fail; None if the task has not failed """ @@ -313,7 +313,7 @@ def error(self) -> Optional[TaskError]: class _TableTaskResult(FlightDataTaskResult): - def __init__(self, table: pyarrow.Table, on_close: Optional[OnCloseCallback] = None) -> None: + def __init__(self, table: pyarrow.Table, on_close: OnCloseCallback | None = None) -> None: super().__init__(single_use_data=False) self._table: pyarrow.Table = table @@ -336,7 +336,7 @@ def _close(self) -> None: class _ReaderTaskResult(FlightDataTaskResult): - def __init__(self, reader: pyarrow.RecordBatchReader, on_close: Optional[OnCloseCallback] = None) -> None: + def __init__(self, reader: pyarrow.RecordBatchReader, on_close: OnCloseCallback | None = None) -> None: super().__init__(single_use_data=True) self._reader = reader diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/temporal_container.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/temporal_container.py index f72284c17..5aeda34ee 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/temporal_container.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/temporal_container.py @@ -3,7 +3,7 @@ import time from collections.abc import Iterator from dataclasses import dataclass -from typing import Any, Callable, Generic, Optional, TypeVar +from typing import Any, Callable, Generic, TypeVar import structlog from readerwriterlock import rwlock @@ -153,7 +153,7 @@ def start_collector(self) -> None: if self._thread.is_alive(): self._thread.start() - def get_entry(self, entry_id: str) -> Optional[T]: + def get_entry(self, entry_id: str) -> T | None: """ :param entry_id: entry identifier :return: entry or None if not found @@ -185,7 +185,7 @@ def evict_entry(self, entry_id: str) -> bool: self._entry_evict_fun(entry.value) return True - def pop_entry(self, entry_id: str) -> Optional[T]: + def pop_entry(self, entry_id: str) -> T | None: """ Pops an entry out of the container. This will take the entry out and will not run the eviction function. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/thread_task_executor.py b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/thread_task_executor.py index a4a23e115..bc2773278 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/thread_task_executor.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/tasks/thread_task_executor.py @@ -6,7 +6,7 @@ from concurrent.futures import CancelledError, Future, ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Optional, Union +from typing import Any, Union import opentelemetry.context as otelctx import pyarrow.flight @@ -47,22 +47,22 @@ class _TaskExecutionStats: time when the task was created """ - run_submitted: Optional[float] = None + run_submitted: float | None = None """ time when task was submitted to thread pool to invoke the run() """ - run_started: Optional[float] = None + run_started: float | None = None """ time when some thread actually started the run() """ - run_completed: Optional[float] = None + run_completed: float | None = None """ time when the run() completed (regardless of the result) """ - completed: Optional[float] = None + completed: float | None = None """ time when all work for task execution completed. if the task was actually run, then this value is same as `run_completed`. if the task failed/was cancelled @@ -187,7 +187,7 @@ def __init__( self._lock = threading.RLock() # all these are protected using the lock - self._result_future: Optional[Future[Union[TaskResult, TaskError]]] = None + self._result_future: Future[Union[TaskResult, TaskError]] | None = None self._completed: threading.Condition = threading.Condition(self._lock) @property @@ -289,7 +289,7 @@ def cancel(self) -> bool: # may not be possible return self._task.cancel() - def wait_for_completion(self, timeout: Optional[float] = None) -> None: + def wait_for_completion(self, timeout: float | None = None) -> None: with self._lock: completed = self._completed.wait(timeout=timeout) @@ -566,7 +566,7 @@ def submit( execution.start() self._metrics.queue_size.set(self._queue_size) - def get_task_submitted_timestamp(self, task_id: str) -> Optional[float]: + def get_task_submitted_timestamp(self, task_id: str) -> float | None: with self._task_lock: execution = self._executions.get(task_id) @@ -574,7 +574,7 @@ def get_task_submitted_timestamp(self, task_id: str) -> Optional[float]: return execution.stats.created return None - def wait_for_result(self, task_id: str, timeout: Optional[float] = None) -> Optional[TaskExecutionResult]: + def wait_for_result(self, task_id: str, timeout: float | None = None) -> TaskExecutionResult | None: with self._task_lock: execution = self._executions.get(task_id) result = self._results.get_entry(task_id) @@ -621,7 +621,7 @@ def close_result(self, task_id: str) -> bool: self._on_finished_task_evicted(result) return True - def stop(self, cancel_running: bool = True, timeout: Optional[float] = None) -> None: + def stop(self, cancel_running: bool = True, timeout: float | None = None) -> None: """ Stops the service. Any pending tasks will be immediately cancelled. Tasks that are already executing are allowed to complete. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/logging.py b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/logging.py index f7d0e9d83..7b96d2a99 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/logging.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/logging.py @@ -5,7 +5,7 @@ import os from logging.config import fileConfig -from typing import Any, Optional, Union +from typing import Any, Union import orjson import structlog @@ -13,7 +13,7 @@ from structlog.typing import EventDict, WrappedLogger -def _resolve_config(logging_ini: str, for_module: Optional[str]) -> str: +def _resolve_config(logging_ini: str, for_module: str | None) -> str: if os.path.isabs(logging_ini): return logging_ini else: @@ -47,7 +47,7 @@ class _OtelTraceContextInjector: __slots__ = ("_trace_id_key", "_span_id_key", "_parent_span_id_key") - def __init__(self, trace_ctx_keys: Optional[dict[str, str]] = None) -> None: + def __init__(self, trace_ctx_keys: dict[str, str] | None = None) -> None: _keys = trace_ctx_keys or {} # do one-time lookup of the actual key names under which the different @@ -76,7 +76,7 @@ def __call__(self, _: WrappedLogger, __: str, event_dict: EventDict) -> EventDic event_dict[self._trace_id_key] = f"{span_ctx.trace_id:x}" event_dict[self._span_id_key] = f"{span_ctx.span_id:x}" - parent_ctx: Optional[trace.SpanContext] = getattr(span, "parent", None) + parent_ctx: trace.SpanContext | None = getattr(span, "parent", None) if parent_ctx: event_dict[self._parent_span_id_key] = f"{parent_ctx.span_id:x}" @@ -87,7 +87,7 @@ def _configure_structlog( dev_log: bool, event_key: str, add_trace_ctx: bool = False, - trace_ctx_keys: Optional[dict[str, str]] = None, + trace_ctx_keys: dict[str, str] | None = None, ) -> None: common_processors: list[Any] = [ structlog.stdlib.filter_by_level, @@ -129,9 +129,9 @@ def init_logging( logging_ini: str, dev_log: bool = False, event_key: str = "event", - for_module: Optional[str] = None, + for_module: str | None = None, add_trace_ctx: bool = False, - trace_ctx_keys: Optional[dict[str, str]] = None, + trace_ctx_keys: dict[str, str] | None = None, ) -> str: """ Initializes python logging from the file on the provided path. If the path is absolute, then it is diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/methods_discovery.py b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/methods_discovery.py index fb2bf6746..1fa7ca379 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/methods_discovery.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/methods_discovery.py @@ -2,7 +2,6 @@ import importlib from functools import wraps from inspect import signature -from typing import Optional from gooddata_flight_server.exceptions import FlightMethodsModuleError from gooddata_flight_server.server.base import ( @@ -74,7 +73,7 @@ def _only_valid_flight_methods_factory( return factory -def get_methods_factory(module_name: str, root: Optional[str] = None) -> FlightServerMethodsFactory: +def get_methods_factory(module_name: str, root: str | None = None) -> FlightServerMethodsFactory: """ Get the method factory from the given module. The module should contain exactly one method decorated with @flight_server_methods. diff --git a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/otel_tracing.py b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/otel_tracing.py index cfa1dc2dc..330d87402 100644 --- a/packages/gooddata-flight-server/src/gooddata_flight_server/utils/otel_tracing.py +++ b/packages/gooddata-flight-server/src/gooddata_flight_server/utils/otel_tracing.py @@ -3,7 +3,6 @@ import platform import socket import sys -from typing import Optional from opentelemetry import trace from opentelemetry.sdk.resources import ( @@ -111,7 +110,7 @@ def _create_resource(config: OtelConfig) -> Resource: ) -def initialize_otel_tracing(config: Optional[OtelConfig]) -> None: +def initialize_otel_tracing(config: OtelConfig | None) -> None: """ Initializes OpenTelemetry tracing according to the provided `config`: diff --git a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py index e33f6d340..9bc8b8b64 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py @@ -1,7 +1,7 @@ # (C) 2021 GoodData Corporation from __future__ import annotations -from typing import Any, Callable, Optional, Union, cast +from typing import Any, Callable, Union, cast from gooddata_sdk import ( Attribute, @@ -33,7 +33,7 @@ class ExecutionDefinitionBuilder: _DEFAULT_INDEX_NAME: str = "0" - def __init__(self, columns: ColumnsDef, index_by: Optional[IndexDef] = None, is_cancellable: bool = False) -> None: + def __init__(self, columns: ColumnsDef, index_by: IndexDef | None = None, is_cancellable: bool = False) -> None: """ Initializes the ExecutionDefinitionBuilder instance with columns and an optional index_by definition. Processes the given columns and index_by @@ -105,7 +105,7 @@ def _add_column(self, column_name: str, column_def: Union[str, Attribute, Metric self._col_to_metric_idx[column_name] = len(self._metrics) self._metrics.append(item) - def _process_index(self, index_by: Optional[IndexDef] = None) -> None: + def _process_index(self, index_by: IndexDef | None = None) -> None: """ Processes the given index definition (index_by) to determine whether to reference attributes or raise an error when attempting to reference metrics. Updates the internal index-to-attribute mapping @@ -186,7 +186,7 @@ def _process_index_item_without_col_ref(self, index_name: str, index_def: LabelI self._index_to_attr_idx[index_name] = len(self._attributes) self._attributes.append(attr_item) - def _update_filter_ids(self, filter_by: Optional[Union[Filter, list[Filter]]] = None) -> Optional[list[Filter]]: + def _update_filter_ids(self, filter_by: Union[Filter, list[Filter]] | None = None) -> list[Filter] | None: """ Updates the filter IDs for the given filters. If a filter references a metric/attribute by a string, it is converted to the corresponding internal ID. @@ -224,9 +224,7 @@ def _update_filter_ids(self, filter_by: Optional[Union[Filter, list[Filter]]] = return filters - def build_execution_definition( - self, filter_by: Optional[Union[Filter, list[Filter]]] = None - ) -> ExecutionDefinition: + def build_execution_definition(self, filter_by: Union[Filter, list[Filter]] | None = None) -> ExecutionDefinition: """ Builds an ExecutionDefinition instance with the current configuration of metrics, attributes, and filters. @@ -262,8 +260,8 @@ def _compute( sdk: GoodDataSdk, workspace_id: str, columns: ColumnsDef, - index_by: Optional[IndexDef] = None, - filter_by: Optional[Union[Filter, list[Filter]]] = None, + index_by: IndexDef | None = None, + filter_by: Union[Filter, list[Filter]] | None = None, is_cancellable: bool = False, ) -> tuple[Execution, dict[str, int], dict[str, int], dict[str, int]]: """ @@ -360,8 +358,8 @@ def _extract_from_attributes_and_maybe_metrics( cols: list[str], col_to_attr_idx: dict[str, int], col_to_metric_idx: dict[str, int], - index_to_attr_idx: Optional[dict[str, int]] = None, - result_page_len: Optional[int] = None, + index_to_attr_idx: dict[str, int] | None = None, + result_page_len: int | None = None, ) -> tuple[dict, dict]: """ Internal function that extracts data from execution response with attributes columns and @@ -424,11 +422,11 @@ def compute_and_extract( sdk: GoodDataSdk, workspace_id: str, columns: ColumnsDef, - index_by: Optional[IndexDef] = None, - filter_by: Optional[Union[Filter, list[Filter]]] = None, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + index_by: IndexDef | None = None, + filter_by: Union[Filter, list[Filter]] | None = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> tuple[dict, dict]: """ Convenience function that computes and extracts data from the execution response. diff --git a/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py b/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py index 196a18555..2074a6526 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py @@ -1,7 +1,7 @@ # (C) 2021 GoodData Corporation from __future__ import annotations -from typing import Callable, Literal, Optional, Union +from typing import Callable, Literal, Union import pandas from gooddata_api_client import models @@ -72,10 +72,10 @@ def indexed( self, index_by: IndexDef, columns: ColumnsDef, - filter_by: Optional[Union[Filter, list[Filter]]] = None, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + filter_by: Union[Filter, list[Filter]] | None = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.DataFrame: """ Creates a data frame indexed by values of the label. The data frame columns will be created from either @@ -115,10 +115,10 @@ def indexed( def not_indexed( self, columns: ColumnsDef, - filter_by: Optional[Union[Filter, list[Filter]]] = None, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + filter_by: Union[Filter, list[Filter]] | None = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.DataFrame: """ Creates a data frame with columns created from metrics and or labels. @@ -152,11 +152,11 @@ def not_indexed( def for_items( self, items: ColumnsDef, - filter_by: Optional[Union[Filter, list[Filter]]] = None, + filter_by: Union[Filter, list[Filter]] | None = None, auto_index: bool = True, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.DataFrame: """ Creates a data frame for named items. This is a convenience method that will create DataFrame with or @@ -214,9 +214,9 @@ def for_visualization( self, visualization_id: str, auto_index: bool = True, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.DataFrame: """ Creates a data frame with columns based on the content of the visualization with the provided identifier. @@ -256,10 +256,10 @@ def for_visualization( def for_created_visualization( self, created_visualizations_response: dict, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, optimized: bool = False, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> tuple[pandas.DataFrame, DataFrameMetadata]: """ Creates a data frame using a created visualization. @@ -305,13 +305,13 @@ def result_cache_metadata_for_exec_result_id(self, result_id: str) -> ResultCach def for_exec_def( self, exec_def: ExecutionDefinition, - label_overrides: Optional[LabelOverrides] = None, + label_overrides: LabelOverrides | None = None, result_size_dimensions_limits: ResultSizeDimensions = (), - result_size_bytes_limit: Optional[int] = None, + result_size_bytes_limit: int | None = None, page_size: int = _DEFAULT_PAGE_SIZE, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + on_execution_submitted: Callable[[Execution], None] | None = None, optimized: bool = False, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> tuple[pandas.DataFrame, DataFrameMetadata]: """ Creates a data frame using an execution definition. @@ -378,15 +378,15 @@ def for_exec_def( def for_exec_result_id( self, result_id: str, - label_overrides: Optional[LabelOverrides] = None, - result_cache_metadata: Optional[ResultCacheMetadata] = None, + label_overrides: LabelOverrides | None = None, + result_cache_metadata: ResultCacheMetadata | None = None, result_size_dimensions_limits: ResultSizeDimensions = (), - result_size_bytes_limit: Optional[int] = None, + result_size_bytes_limit: int | None = None, use_local_ids_in_headers: bool = False, use_primary_labels_in_attributes: bool = False, page_size: int = _DEFAULT_PAGE_SIZE, optimized: bool = False, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> tuple[pandas.DataFrame, DataFrameMetadata]: """ Retrieves a DataFrame and DataFrame metadata for a given execution result identifier. diff --git a/packages/gooddata-pandas/src/gooddata_pandas/good_pandas.py b/packages/gooddata-pandas/src/gooddata_pandas/good_pandas.py index 1ae224068..23a99b9ca 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/good_pandas.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/good_pandas.py @@ -2,7 +2,6 @@ from __future__ import annotations from pathlib import Path -from typing import Optional from gooddata_sdk import GoodDataSdk from gooddata_sdk.utils import PROFILES_FILE_PATH, good_pandas_profile_content @@ -24,8 +23,8 @@ def __init__( self, host: str, token: str, - headers_host: Optional[str] = None, - **custom_headers_: Optional[str], + headers_host: str | None = None, + **custom_headers_: str | None, ) -> None: """ Initializes GoodPandas instance. diff --git a/packages/gooddata-pandas/src/gooddata_pandas/result_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/result_convertor.py index 46a3959d0..fdc953979 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/result_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/result_convertor.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterator from functools import cached_property -from typing import Any, Callable, Literal, Optional, Union, cast +from typing import Any, Callable, Literal, Union, cast import pandas from attrs import define, field, frozen @@ -31,7 +31,7 @@ class _Header(ABC): def _dict(self) -> dict[str, Any]: pass - def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]: + def get(self, key: str, default: Any | None = None) -> Any | None: return self._dict.get(key, default) @@ -89,7 +89,7 @@ def _dict(self) -> dict[str, Any]: return {"totalHeader": {"function": self.function}} -def _header_from_dict(d: dict[str, Any]) -> Optional[_Header]: +def _header_from_dict(d: dict[str, Any]) -> _Header | None: """ Convert dict representation to _Header object. :param d: dictionary representation of a header @@ -167,7 +167,7 @@ def __getitem__(self, index: int) -> _Header: # Optimized version of _DataWithHeaders uses _HeaderContainer instead of list of headers _HeadersByAxis = tuple[ - Union[_DataHeaders, _DataHeaderContainers], Union[Optional[_DataHeaders], Optional[_DataHeaderContainers]] + Union[_DataHeaders, _DataHeaderContainers], Union[_DataHeaders | None, _DataHeaderContainers | None] ] @@ -188,8 +188,8 @@ class _DataWithHeaders: data: list[_DataArray] data_headers: _HeadersByAxis - grand_totals: tuple[Optional[list[_DataArray]], Optional[list[_DataArray]]] - grand_total_headers: tuple[Optional[list[dict[str, _DataHeaders]]], Optional[list[dict[str, _DataHeaders]]]] + grand_totals: tuple[list[_DataArray] | None, list[_DataArray] | None] + grand_total_headers: tuple[list[dict[str, _DataHeaders]] | None, list[dict[str, _DataHeaders]] | None] @define @@ -209,12 +209,10 @@ class _AbstractAccumulatedData(ABC): """ data: list[_DataArray] = field(init=False, factory=list) - data_headers: list[Optional[Any]] = field(init=False, factory=lambda: [None, None]) - grand_totals: list[Optional[list[_DataArray]]] = field(init=False, factory=lambda: [None, None]) + data_headers: list[Any | None] = field(init=False, factory=lambda: [None, None]) + grand_totals: list[list[_DataArray] | None] = field(init=False, factory=lambda: [None, None]) total_of_grant_totals_processed: bool = field(init=False, default=False) - grand_totals_headers: list[Optional[list[dict[str, _DataHeaders]]]] = field( - init=False, factory=lambda: [None, None] - ) + grand_totals_headers: list[list[dict[str, _DataHeaders]] | None] = field(init=False, factory=lambda: [None, None]) def accumulate_data(self, from_result: ExecutionResult) -> None: """ @@ -473,7 +471,7 @@ def from_data( ) @staticmethod - def _get_totals_indexes(headers: Optional[Any]) -> list[list[int]]: + def _get_totals_indexes(headers: Any | None) -> list[list[int]]: if headers is None: return [] return [ @@ -486,7 +484,7 @@ def _read_complete_execution_result( execution_response: BareExecutionResponse, result_cache_metadata: ResultCacheMetadata, result_size_dimensions_limits: ResultSizeDimensions, - result_size_bytes_limit: Optional[int] = None, + result_size_bytes_limit: int | None = None, page_size: int = _DEFAULT_PAGE_SIZE, optimized: bool = False, ) -> _DataWithHeaders: @@ -569,10 +567,10 @@ def _create_header_mapper( response: BareExecutionResponse, dim: int, primary_attribute_labels_mapping: dict[int, dict[str, str]], - label_overrides: Optional[LabelOverrides] = None, + label_overrides: LabelOverrides | None = None, use_local_ids_in_headers: bool = False, use_primary_labels_in_attributes: bool = False, -) -> Callable[[Any, Optional[int]], Optional[str]]: +) -> Callable[[Any, int | None], str | None]: """ Prepares a header mapper function which translates header structures into appropriate labels used in a dataframe. @@ -597,7 +595,7 @@ def _create_header_mapper( attribute_labels = label_overrides.get("labels", {}) measure_labels = label_overrides.get("metrics", {}) - def _mapper(header: Union[dict, _Header, None], header_idx: Optional[int]) -> Optional[str]: + def _mapper(header: Union[dict, _Header, None], header_idx: int | None) -> str | None: label = None if header is None: pass @@ -655,7 +653,7 @@ def _headers_to_index( label_overrides: LabelOverrides, use_local_ids_in_headers: bool = False, use_primary_labels_in_attributes: bool = False, -) -> tuple[Optional[pandas.Index], dict[int, dict[str, str]]]: +) -> tuple[pandas.Index | None, dict[int, dict[str, str]]]: """Converts headers to a pandas MultiIndex. This function converts the headers present in the response to a pandas MultiIndex (can be used in pandas dataframes) @@ -700,7 +698,7 @@ def _headers_to_index( def _merge_grand_totals_into_data( extract: _DataWithHeaders, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> Union[_DataArray, list[_DataArray]]: """ Merges grand totals into the extracted data. This function will mutate the extracted data, @@ -767,12 +765,12 @@ def convert_execution_response_to_dataframe( result_cache_metadata: ResultCacheMetadata, label_overrides: LabelOverrides, result_size_dimensions_limits: ResultSizeDimensions, - result_size_bytes_limit: Optional[int] = None, + result_size_bytes_limit: int | None = None, use_local_ids_in_headers: bool = False, use_primary_labels_in_attributes: bool = False, page_size: int = _DEFAULT_PAGE_SIZE, optimized: bool = False, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> tuple[pandas.DataFrame, DataFrameMetadata]: """ Converts execution result to a pandas dataframe, maintaining the dimensionality of the result. diff --git a/packages/gooddata-pandas/src/gooddata_pandas/series.py b/packages/gooddata-pandas/src/gooddata_pandas/series.py index e7e938a8a..664f17186 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/series.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/series.py @@ -1,7 +1,7 @@ # (C) 2021 GoodData Corporation from __future__ import annotations -from typing import Callable, Optional, Union +from typing import Callable, Union import pandas from gooddata_sdk import Attribute, Execution, Filter, GoodDataSdk, ObjId, SimpleMetric @@ -27,10 +27,10 @@ def indexed( self, index_by: IndexDef, data_by: Union[SimpleMetric, str, ObjId, Attribute], - filter_by: Optional[Union[Filter, list[Filter]]] = None, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + filter_by: Union[Filter, list[Filter]] | None = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.Series: """Creates pandas Series from data points calculated from a single `data_by`. @@ -94,11 +94,11 @@ def indexed( def not_indexed( self, data_by: Union[SimpleMetric, str, ObjId, Attribute], - granularity: Optional[Union[list[LabelItemDef], IndexDef]] = None, - filter_by: Optional[Union[Filter, list[Filter]]] = None, - on_execution_submitted: Optional[Callable[[Execution], None]] = None, + granularity: Union[list[LabelItemDef], IndexDef] | None = None, + filter_by: Union[Filter, list[Filter]] | None = None, + on_execution_submitted: Callable[[Execution], None] | None = None, is_cancellable: bool = False, - result_page_len: Optional[int] = None, + result_page_len: int | None = None, ) -> pandas.Series: """ Creates a pandas.Series from data points calculated from a single `data_by` without constructing an index. @@ -136,7 +136,7 @@ def not_indexed( """ if isinstance(granularity, list): - _index: Optional[IndexDef] = {str(idx): label for idx, label in enumerate(granularity)} + _index: IndexDef | None = {str(idx): label for idx, label in enumerate(granularity)} else: _index = granularity diff --git a/packages/gooddata-pandas/src/gooddata_pandas/utils.py b/packages/gooddata-pandas/src/gooddata_pandas/utils.py index fb8fd8b16..f8789925e 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/utils.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/utils.py @@ -3,7 +3,7 @@ import hashlib import uuid -from typing import Any, Optional, Union +from typing import Any, Union import pandas from gooddata_sdk import ( @@ -74,7 +74,7 @@ def _val_to_hash(val: str) -> str: return hash_object.hexdigest() -def _str_to_obj_id(val: DataItemDef) -> Optional[ObjId]: +def _str_to_obj_id(val: DataItemDef) -> ObjId | None: """ Convert a value to ObjId if it is in one of ["label", "metric", "fact"] types. @@ -110,7 +110,7 @@ def _try_obj_id(val: DataItemDef) -> DataItemDef: return val -def _to_item(val: DataItemDef, local_id: Optional[str] = None) -> Union[Attribute, Metric]: +def _to_item(val: DataItemDef, local_id: str | None = None) -> Union[Attribute, Metric]: """ Convert a DataItemDef value to either an Attribute or a Metric based on its type. @@ -136,7 +136,7 @@ def _to_item(val: DataItemDef, local_id: Optional[str] = None) -> Union[Attribut raise ValueError(f"Invalid column_by item {val}") -def _to_attribute(val: LabelItemDef, local_id: Optional[str] = None) -> Attribute: +def _to_attribute(val: LabelItemDef, local_id: str | None = None) -> Attribute: """ Convert a LabelItemDef value to an Attribute. @@ -169,7 +169,7 @@ def _typed_attribute_value(ct_attr: CatalogAttribute, value: Any) -> Any: return converter.to_external_type(value) -def make_pandas_index(index: dict) -> Optional[Union[Index, MultiIndex]]: +def make_pandas_index(index: dict) -> Union[Index, MultiIndex] | None: """ Create a pandas index or multi-index based on the input index dictionary. diff --git a/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def.py b/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def.py index fb2a8de61..a131657c0 100644 --- a/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def.py +++ b/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def.py @@ -1,6 +1,6 @@ # (C) 2022 GoodData Corporation from pathlib import Path -from typing import Literal, Optional +from typing import Literal import pytest from gooddata_pandas import DataFrameFactory @@ -27,11 +27,11 @@ def _run_and_validate_results( gdf: DataFrameFactory, exec_def: ExecutionDefinition, expected: tuple[int, int], - expected_row_totals: Optional[list[list[int]]] = None, - expected_column_totals: Optional[list[list[int]]] = None, + expected_row_totals: list[list[int]] | None = None, + expected_column_totals: list[list[int]] | None = None, page_size: int = 100, optimized: bool = False, - grand_totals_position: Optional[Literal["pinnedBottom", "pinnedTop", "bottom", "top"]] = "bottom", + grand_totals_position: Literal["pinnedBottom", "pinnedTop", "bottom", "top"] | None = "bottom", ) -> str: # generate dataframe from exec_def result, result_metadata = gdf.for_exec_def( diff --git a/packages/tests-support/src/tests_support/vcrpy_utils.py b/packages/tests-support/src/tests_support/vcrpy_utils.py index d645cb433..73450d61f 100644 --- a/packages/tests-support/src/tests_support/vcrpy_utils.py +++ b/packages/tests-support/src/tests_support/vcrpy_utils.py @@ -5,7 +5,7 @@ import os import typing from json import JSONDecodeError -from typing import Any, Optional +from typing import Any import vcr import yaml @@ -88,8 +88,8 @@ def custom_before_request(request, headers_str: str = HEADERS_STR): def custom_before_response( response: dict[str, Any], headers_str: str = HEADERS_STR, - non_static_headers: Optional[list[str]] = None, - placeholder: Optional[list[str]] = None, + non_static_headers: list[str] | None = None, + placeholder: list[str] | None = None, ): if non_static_headers is None: non_static_headers = NON_STATIC_HEADERS diff --git a/pyproject.toml b/pyproject.toml index 51bedf25a..c45348052 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,8 +106,6 @@ lint.ignore = [ "SIM105", # ignore the rule (import from `collections.abc` instead: `Callable`) "UP035", - # ignore the rule (use `X | Y` in `isinstance` call instead of `(X, Y)`) - "UP038", ] target-version = "py310" From 3e9d0acb216f3da0f1aae0e2a5dfd25155e7f71c Mon Sep 17 00:00:00 2001 From: Jan Kadlec Date: Tue, 3 Mar 2026 16:33:06 +0100 Subject: [PATCH 2/2] feat: add support for GEO_AREA label GEO_AREA label was missing in gooddata-dbt and needs to be added. JIRA: TRIVIAL risk: low --- .../gooddata-dbt/src/gooddata_dbt/dbt/base.py | 1 + .../src/gooddata_dbt/dbt/tables.py | 24 ++++++++++--------- .../tests/resources/dbt_target/manifest.json | 7 +++++- packages/gooddata-dbt/tests/test_tables.py | 4 ++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py index 3b42c2867..c89f0f127 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py @@ -21,6 +21,7 @@ class GoodDataLabelType(Enum): HYPERLINK = "HYPERLINK" GEO_LATITUDE = "GEO_LATITUDE" GEO_LONGITUDE = "GEO_LONGITUDE" + GEO_AREA = "GEO_AREA" class GoodDataSortDirection(Enum): diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py index 36f1dcddf..37746089f 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py @@ -40,6 +40,7 @@ class DbtModelMetaGoodDataColumnProps(Base): sort_column: str | None = None sort_direction: GoodDataSortDirection | None = None default_view: bool | None = None + geo_area_config: dict[str, dict[str, str]] | None = None @property def gooddata_ref_table_ldm_id(self) -> str | None: @@ -392,17 +393,18 @@ def make_labels(table: DbtModelTable, attribute_column: DbtModelColumn) -> tuple default_view = None for column in table.columns.values(): if column.gooddata_is_label(attribute_column.name): - labels.append( - { - "id": column.ldm_id, - "title": column.gooddata_ldm_title, - "description": column.gooddata_ldm_description, - "source_column": column.name, - "source_column_data_type": column.data_type, - "value_type": column.label_type, - "tags": [table.gooddata_ldm_title] + column.tags, - } - ) + label_dict: dict = { + "id": column.ldm_id, + "title": column.gooddata_ldm_title, + "description": column.gooddata_ldm_description, + "source_column": column.name, + "source_column_data_type": column.data_type, + "value_type": column.label_type, + "tags": [table.gooddata_ldm_title] + column.tags, + } + if column.meta.gooddata.geo_area_config is not None: + label_dict["geo_area_config"] = column.meta.gooddata.geo_area_config + labels.append(label_dict) if column.meta.gooddata.default_view: default_view = { "id": column.ldm_id, diff --git a/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json b/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json index 582366284..57d50cc57 100644 --- a/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json +++ b/packages/gooddata-dbt/tests/resources/dbt_target/manifest.json @@ -1868,7 +1868,12 @@ "gooddata": { "ldm_type": "label", "label_type": "GEO_AREA", - "attribute_column": "code" + "attribute_column": "code", + "geo_area_config": { + "collection": { + "id": "countries" + } + } } }, "data_type": null, diff --git a/packages/gooddata-dbt/tests/test_tables.py b/packages/gooddata-dbt/tests/test_tables.py index fcfc74e62..c0c9bb9b5 100644 --- a/packages/gooddata-dbt/tests/test_tables.py +++ b/packages/gooddata-dbt/tests/test_tables.py @@ -77,3 +77,7 @@ def test_make_ldm_geo_area(): assert label_types["latitude_origin"] == "GEO_LATITUDE" assert label_types["longitude_origin"] == "GEO_LONGITUDE" assert label_types["country_origin"] == "GEO_AREA" + # Verify geo_area_config is propagated + country_label = next(lbl for lbl in labels if lbl.id == "country_origin") + assert country_label.geo_area_config is not None + assert country_label.geo_area_config.collection.id == "countries"