Index

_

__post_init__() (astro.sql.Table method) (astro.sql.table.Table method) __repr__() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.files.base.File method) (astro.files.File method) (astro.files.locations.base.BaseFileLocation method) (astro.files.locations.BaseFileLocation method) (astro.files.types.base.FileType method) (astro.files.types.FileType method) __str__() (astro.files.base.File method) (astro.files.File method) (astro.files.locations.base.BaseFileLocation method) (astro.files.locations.BaseFileLocation method) (astro.files.types.base.FileType method) (astro.files.types.FileType method) __version__ (in module astro) _build_merge_sql() (astro.databases.snowflake.SnowflakeDatabase method) _create_schema_statement (astro.databases.base.BaseDatabase attribute) (astro.databases.BaseDatabase attribute)

_create_table_statement (astro.databases.base.BaseDatabase attribute) (astro.databases.BaseDatabase attribute) _create_unique_table_name() (astro.sql.Table method) (astro.sql.table.Table method) _drop_table_statement (astro.databases.base.BaseDatabase attribute) (astro.databases.BaseDatabase attribute) _find_first_table_from_op_args() (in module astro.utils.table) _get_dataframe() (in module astro.sql.operators.dataframe) _get_executor_from_job_id() (astro.sql.CleanupOperator static method) (astro.sql.operators.cleanup.CleanupOperator static method) _is_dag_running() (astro.sql.CleanupOperator method) (astro.sql.operators.cleanup.CleanupOperator method) _is_single_worker_mode() (astro.sql.CleanupOperator class method) (astro.sql.operators.cleanup.CleanupOperator class method) _name (astro.sql.Table attribute) (astro.sql.table.Table attribute) _parse_s3_env_var() (astro.files.locations.amazon.s3.S3Location static method) _populate_normalize_config() (astro.sql.operators.load_file.LoadFile static method) _pull_first_table_from_op_kwargs() (in module astro.utils.table) _pull_first_table_from_parameters() (in module astro.utils.table)

A

append() (in module astro.sql) APPEND_COLUMN_TYPE (in module astro.sql) (in module astro.sql.operators.append) append_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) AppendOperator (class in astro.sql) (class in astro.sql.operators.append) astro module astro.constants module astro.databases module astro.databases.base module astro.databases.google module astro.databases.google.bigquery module astro.databases.postgres module astro.databases.snowflake module astro.databases.sqlite module astro.exceptions module astro.files module astro.files.base module astro.files.locations module astro.files.locations.amazon module astro.files.locations.amazon.s3 module astro.files.locations.base module astro.files.locations.google module astro.files.locations.google.gcs module astro.files.locations.http module astro.files.locations.local module astro.files.types module

astro.files.types.base module astro.files.types.csv module astro.files.types.json module astro.files.types.ndjson module astro.files.types.parquet module astro.settings module astro.sql module astro.sql.operators module astro.sql.operators.append module astro.sql.operators.base module astro.sql.operators.cleanup module astro.sql.operators.dataframe module astro.sql.operators.export_file module astro.sql.operators.load_file module astro.sql.operators.merge module astro.sql.operators.raw_sql module astro.sql.operators.transform module astro.sql.operators.truncate module astro.sql.table module astro.utils module astro.utils.load module astro.utils.path module astro.utils.table module astro.utils.task_id_helper module

B

BaseDatabase (class in astro.databases) (class in astro.databases.base) BaseFileLocation (class in astro.files.locations) (class in astro.files.locations.base)

BaseSQLOperator (class in astro.sql.operators.base) BIGQUERY (astro.constants.Database attribute) BigqueryDatabase (class in astro.databases.google.bigquery)

C

check_if_connection_exists() (in module astro.files) check_non_existing_local_file_path() (astro.files.locations.base.BaseFileLocation static method) (astro.files.locations.BaseFileLocation static method) cleanup() (in module astro.sql) CleanupOperator (class in astro.sql) (class in astro.sql.operators.cleanup) columns (astro.sql.Table attribute) (astro.sql.table.Table attribute) conn_id (astro.files.base.File property) (astro.files.File property) (astro.sql.Table attribute) (astro.sql.table.Table attribute) CONN_TYPE_TO_MODULE_PATH (in module astro.databases) connection (astro.databases.base.BaseDatabase property) (astro.databases.BaseDatabase property) copy_remote_file_to_local() (in module astro.utils.load) create_database() (in module astro.databases) create_file_location() (in module astro.files.locations) create_file_type() (in module astro.files.types) create_from_dataframe() (astro.files.base.File method) (astro.files.File method) (astro.files.types.base.FileType method) (astro.files.types.csv.CSVFileType method) (astro.files.types.CSVFileType method) (astro.files.types.FileType method) (astro.files.types.json.JSONFileType method) (astro.files.types.JSONFileType method) (astro.files.types.ndjson.NDJSONFileType method) (astro.files.types.NDJSONFileType method) (astro.files.types.parquet.ParquetFileType method) (astro.files.types.ParquetFileType method)

create_output_table_if_needed() (astro.sql.operators.base.BaseSQLOperator method) create_schema_if_needed() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.sqlite.SqliteDatabase method) create_similar_table() (astro.sql.Table method) (astro.sql.table.Table method) create_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) create_table_from_select_statement() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) CSV (astro.constants.FileType attribute) (astro.files.types.FileTypeConstants attribute) CSVFileType (class in astro.files.types) (class in astro.files.types.csv) CUSTOM_CONN_TYPE_TO_MODULE_PATH (in module astro.databases)

D

database (astro.sql.table.Metadata attribute) Database (class in astro.constants) database_impl (astro.sql.operators.base.BaseSQLOperator attribute) dataframe() (in module astro.sql) DataframeOperator (class in astro.sql) (class in astro.sql.operators.dataframe) DEFAULT_CHUNK_SIZE (in module astro.constants) DEFAULT_CONN_ID (in module astro.databases.google.bigquery) (in module astro.databases.postgres) (in module astro.databases.snowflake) (in module astro.databases.sqlite) DEFAULT_CONN_TYPE_TO_MODULE_PATH (in module astro.databases) (in module astro.files.locations)

default_metadata (astro.databases.base.BaseDatabase property) (astro.databases.BaseDatabase property) (astro.databases.google.bigquery.BigqueryDatabase property) (astro.databases.postgres.PostgresDatabase property) (astro.databases.snowflake.SnowflakeDatabase property) (astro.databases.sqlite.SqliteDatabase property) DEFAULT_SCHEMA (in module astro.settings) drop_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.sql.CleanupOperator method) (astro.sql.operators.cleanup.CleanupOperator method)

E

ensure_internal_quotes_closed() (in module astro.databases.snowflake) ensure_only_valid_characters() (in module astro.databases.snowflake) execute() (astro.sql.AppendOperator method) (astro.sql.CleanupOperator method) (astro.sql.DataframeOperator method) (astro.sql.MergeOperator method) (astro.sql.operators.append.AppendOperator method) (astro.sql.operators.base.BaseSQLOperator method) (astro.sql.operators.cleanup.CleanupOperator method) (astro.sql.operators.dataframe.DataframeOperator method) (astro.sql.operators.export_file.ExportFile method) (astro.sql.operators.load_file.LoadFile method) (astro.sql.operators.merge.MergeOperator method) (astro.sql.operators.raw_sql.RawSQLOperator method) (astro.sql.operators.transform.TransformOperator method) (astro.sql.operators.truncate.TruncateOperator method) (astro.sql.RawSQLOperator method) (astro.sql.TransformOperator method) (astro.sql.TruncateOperator method) exists() (astro.files.base.File method) (astro.files.File method) (astro.files.locations.base.BaseFileLocation method) (astro.files.locations.BaseFileLocation method)

export_file() (in module astro.sql) (in module astro.sql.operators.export_file) export_table_to_file() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) export_table_to_pandas_dataframe() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) export_to_dataframe() (astro.files.base.File method) (astro.files.File method) (astro.files.types.base.FileType method) (astro.files.types.csv.CSVFileType method) (astro.files.types.CSVFileType method) (astro.files.types.FileType method) (astro.files.types.json.JSONFileType method) (astro.files.types.JSONFileType method) (astro.files.types.ndjson.NDJSONFileType method) (astro.files.types.NDJSONFileType method) (astro.files.types.parquet.ParquetFileType method) (astro.files.types.ParquetFileType method) ExportExistsStrategy (in module astro.constants) ExportFile (class in astro.sql.operators.export_file)

F

File (class in astro.files) (class in astro.files.base) FileLocation (class in astro.constants) (class in astro.files.locations) FileType (class in astro.constants) (class in astro.files.types) (class in astro.files.types.base)

FileTypeConstants (class in astro.files.types) filter_for_temp_tables() (in module astro.sql.operators.cleanup) find_first_table() (in module astro.utils.table) flatten() (astro.files.types.ndjson.NDJSONFileType static method) (astro.files.types.NDJSONFileType static method)

G

GCSLocation (class in astro.files.locations.google.gcs) get_all_task_outputs() (astro.sql.CleanupOperator method) (astro.sql.operators.cleanup.CleanupOperator method) get_class_name() (in module astro.databases) (in module astro.files.locations) (in module astro.utils.path) get_dict_with_module_names_to_dot_notations() (in module astro.databases) (in module astro.files.locations) (in module astro.utils.path) get_files() (in module astro.files) (in module astro.files.base) get_filetype() (in module astro.files.types) get_location_type() (astro.files.locations.base.BaseFileLocation static method) (astro.files.locations.BaseFileLocation static method) get_merge_initialization_query() (astro.databases.base.BaseDatabase static method) (astro.databases.BaseDatabase static method) (astro.databases.google.bigquery.BigqueryDatabase static method) (astro.databases.sqlite.SqliteDatabase static method)

get_module_dot_notation() (in module astro.utils.path) get_provider_info() (in module astro) get_sqla_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.sqlite.SqliteDatabase method) get_sqlalchemy_template_table_identifier_and_parameter() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.snowflake.SnowflakeDatabase method) get_table_qualified_name() (astro.databases.base.BaseDatabase static method) (astro.databases.BaseDatabase static method) (astro.databases.postgres.PostgresDatabase static method) (astro.databases.snowflake.SnowflakeDatabase static method) (astro.databases.sqlite.SqliteDatabase static method) get_task_id() (in module astro.utils.task_id_helper) GS (astro.constants.FileLocation attribute) (astro.files.locations.FileLocation attribute)

H

hook (astro.databases.base.BaseDatabase property) (astro.databases.BaseDatabase property) (astro.databases.google.bigquery.BigqueryDatabase property) (astro.databases.postgres.PostgresDatabase property) (astro.databases.snowflake.SnowflakeDatabase property) (astro.databases.sqlite.SqliteDatabase property)

HTTP (astro.constants.FileLocation attribute) (astro.files.locations.FileLocation attribute) HTTPLocation (class in astro.files.locations.http) HTTPS (astro.constants.FileLocation attribute) (astro.files.locations.FileLocation attribute)

I

illegal_column_name_chars (astro.databases.base.BaseDatabase attribute) (astro.databases.BaseDatabase attribute) (astro.databases.google.bigquery.BigqueryDatabase attribute) (astro.databases.postgres.PostgresDatabase attribute) illegal_column_name_chars_replacement (astro.databases.base.BaseDatabase attribute) (astro.databases.BaseDatabase attribute) (astro.databases.google.bigquery.BigqueryDatabase attribute) (astro.databases.postgres.PostgresDatabase attribute)

is_binary() (astro.files.base.File method) (astro.files.File method) is_empty() (astro.sql.table.Metadata method) is_valid_path() (astro.files.locations.base.BaseFileLocation static method) (astro.files.locations.BaseFileLocation static method) is_valid_snow_identifier() (in module astro.databases.snowflake)

J

JSON (astro.constants.FileType attribute) (astro.files.types.FileTypeConstants attribute)

JSONFileType (class in astro.files.types) (class in astro.files.types.json)

L

load_data() (astro.sql.operators.load_file.LoadFile method) load_data_to_dataframe() (astro.sql.operators.load_file.LoadFile method) load_data_to_table() (astro.sql.operators.load_file.LoadFile method) load_file() (in module astro.sql) (in module astro.sql.operators.load_file) load_file_to_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) load_op_arg_dataframes_into_sql() (in module astro.sql.operators.base) load_op_arg_table_into_dataframe() (in module astro.sql.operators.dataframe) load_op_kwarg_dataframes_into_sql() (in module astro.sql.operators.base) load_op_kwarg_table_into_dataframe() (in module astro.sql.operators.dataframe) load_pandas_dataframe_to_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.google.bigquery.BigqueryDatabase method) (astro.databases.postgres.PostgresDatabase method) (astro.databases.snowflake.SnowflakeDatabase method)

LoadExistStrategy (in module astro.constants) LoadFile (class in astro.sql.operators.load_file) LOCAL (astro.constants.FileLocation attribute) (astro.files.locations.FileLocation attribute) LocalLocation (class in astro.files.locations.local) location_type (astro.files.locations.amazon.s3.S3Location attribute) (astro.files.locations.base.BaseFileLocation property) (astro.files.locations.BaseFileLocation property) (astro.files.locations.google.gcs.GCSLocation attribute) (astro.files.locations.http.HTTPLocation attribute) (astro.files.locations.local.LocalLocation attribute)

M

MAX_TABLE_NAME_LENGTH (in module astro.sql.table) merge() (in module astro.sql) MERGE_COLUMN_TYPE (in module astro.sql) (in module astro.sql.operators.merge) merge_table() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.google.bigquery.BigqueryDatabase method) (astro.databases.postgres.PostgresDatabase method) (astro.databases.snowflake.SnowflakeDatabase method) (astro.databases.sqlite.SqliteDatabase method) MergeConflictStrategy (in module astro.constants) (in module astro.sql) MergeOperator (class in astro.sql) (class in astro.sql.operators.merge) metadata (astro.sql.Table attribute) (astro.sql.table.Table attribute) Metadata (class in astro.sql.table) module astro astro.constants astro.databases astro.databases.base astro.databases.google astro.databases.google.bigquery astro.databases.postgres astro.databases.snowflake astro.databases.sqlite astro.exceptions astro.files astro.files.base astro.files.locations astro.files.locations.amazon astro.files.locations.amazon.s3 astro.files.locations.base astro.files.locations.google astro.files.locations.google.gcs astro.files.locations.http astro.files.locations.local astro.files.types astro.files.types.base astro.files.types.csv astro.files.types.json astro.files.types.ndjson astro.files.types.parquet astro.settings astro.sql astro.sql.operators astro.sql.operators.append astro.sql.operators.base astro.sql.operators.cleanup astro.sql.operators.dataframe astro.sql.operators.export_file astro.sql.operators.load_file astro.sql.operators.merge astro.sql.operators.raw_sql astro.sql.operators.transform astro.sql.operators.truncate astro.sql.table astro.utils astro.utils.load astro.utils.path astro.utils.table astro.utils.task_id_helper

move_function_params_into_sql_params() (astro.sql.operators.base.BaseSQLOperator method)

N

name (astro.files.types.base.FileType property) (astro.files.types.csv.CSVFileType property) (astro.files.types.CSVFileType property) (astro.files.types.FileType property) (astro.files.types.json.JSONFileType property) (astro.files.types.JSONFileType property) (astro.files.types.ndjson.NDJSONFileType property) (astro.files.types.NDJSONFileType property) (astro.files.types.parquet.ParquetFileType property) (astro.files.types.ParquetFileType property) (astro.sql.Table attribute) (astro.sql.Table property) (astro.sql.table.Table attribute) (astro.sql.table.Table property)

NDJSON (astro.constants.FileType attribute) (astro.files.types.FileTypeConstants attribute) NDJSONFileType (class in astro.files.types) (class in astro.files.types.ndjson)

P

PARQUET (astro.constants.FileType attribute) (astro.files.types.FileTypeConstants attribute) ParquetFileType (class in astro.files.types) (class in astro.files.types.parquet) path (astro.files.base.File property) (astro.files.File property) paths (astro.files.locations.amazon.s3.S3Location property) (astro.files.locations.base.BaseFileLocation property) (astro.files.locations.BaseFileLocation property) (astro.files.locations.google.gcs.GCSLocation property) (astro.files.locations.http.HTTPLocation property) (astro.files.locations.local.LocalLocation property)

populate_table_metadata() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.sqlite.SqliteDatabase method) POSTGRES (astro.constants.Database attribute) PostgresDatabase (class in astro.databases.postgres) POSTGRESQL (astro.constants.Database attribute) PYPI_PROJECT_NAME (in module astro.constants)

R

RawSQLOperator (class in astro.sql) (class in astro.sql.operators.raw_sql) read_sql_from_function() (astro.sql.operators.base.BaseSQLOperator method) resolve_tables_from_tasks() (astro.sql.CleanupOperator method) (astro.sql.operators.cleanup.CleanupOperator method)

run_raw_sql() (in module astro.sql) run_sql() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method)

S

S3 (astro.constants.FileLocation attribute) (astro.files.locations.FileLocation attribute) S3Location (class in astro.files.locations.amazon.s3) schema (astro.sql.table.Metadata attribute) SCHEMA (in module astro.settings) schema_exists() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.google.bigquery.BigqueryDatabase method) (astro.databases.postgres.PostgresDatabase method) (astro.databases.snowflake.SnowflakeDatabase method) (astro.databases.sqlite.SqliteDatabase method) size (astro.files.base.File property) (astro.files.File property) (astro.files.locations.amazon.s3.S3Location property) (astro.files.locations.base.BaseFileLocation property) (astro.files.locations.BaseFileLocation property) (astro.files.locations.google.gcs.GCSLocation property) (astro.files.locations.http.HTTPLocation property) (astro.files.locations.local.LocalLocation property)

SNOWFLAKE (astro.constants.Database attribute) SnowflakeDatabase (class in astro.databases.snowflake) sql_type (astro.databases.base.BaseDatabase property) (astro.databases.BaseDatabase property) (astro.databases.google.bigquery.BigqueryDatabase property) (astro.databases.postgres.PostgresDatabase property) (astro.databases.snowflake.SnowflakeDatabase property) (astro.databases.sqlite.SqliteDatabase property) sqlalchemy_engine (astro.databases.base.BaseDatabase property) (astro.databases.BaseDatabase property) (astro.databases.google.bigquery.BigqueryDatabase property) (astro.databases.sqlite.SqliteDatabase property) sqlalchemy_metadata (astro.sql.Table property) (astro.sql.table.Table property) SQLITE (astro.constants.Database attribute) SqliteDatabase (class in astro.databases.sqlite) SUPPORTED_DATABASES (in module astro.constants) (in module astro.databases) SUPPORTED_FILE_LOCATIONS (in module astro.constants) SUPPORTED_FILE_TYPES (in module astro.constants)

T

Table (class in astro.sql) (class in astro.sql.table) table_exists() (astro.databases.base.BaseDatabase method) (astro.databases.BaseDatabase method) (astro.databases.postgres.PostgresDatabase method) temp (astro.sql.Table attribute) (astro.sql.table.Table attribute) TEMP_PREFIX (in module astro.sql.table) template_fields (astro.files.base.File attribute) (astro.files.File attribute) (astro.files.locations.base.BaseFileLocation attribute) (astro.files.locations.BaseFileLocation attribute) (astro.sql.AppendOperator attribute) (astro.sql.CleanupOperator attribute) (astro.sql.MergeOperator attribute) (astro.sql.operators.append.AppendOperator attribute) (astro.sql.operators.cleanup.CleanupOperator attribute) (astro.sql.operators.export_file.ExportFile attribute) (astro.sql.operators.load_file.LoadFile attribute) (astro.sql.operators.merge.MergeOperator attribute) (astro.sql.Table attribute) (astro.sql.table.Table attribute)

transform() (in module astro.sql) TransformOperator (class in astro.sql) (class in astro.sql.operators.transform) translate_jinja_to_sqlalchemy_template() (astro.sql.operators.base.BaseSQLOperator method) transport_params (astro.files.locations.amazon.s3.S3Location property) (astro.files.locations.base.BaseFileLocation property) (astro.files.locations.BaseFileLocation property) (astro.files.locations.google.gcs.GCSLocation property) truncate() (in module astro.sql) TruncateOperator (class in astro.sql) (class in astro.sql.operators.truncate)

W

wait_for_dag_to_finish() (astro.sql.CleanupOperator method) (astro.sql.operators.cleanup.CleanupOperator method)

wrap_identifier() (in module astro.databases.snowflake)