- Data loader
- Data loader and exporter configurations
Data loader
Data loader and exporter configurations
This YAML file contains the information and credentials required to access databases, data warehouses, and data lakes.
io_config.yaml
version: 0.1.1
default:
# Default profile created for data IO access.
# Add your credentials for the source you use, and delete the rest.
AWS_ACCESS_KEY_ID: "{{ env_var('AWS_ACCESS_KEY_ID') }}"
AWS_SECRET_ACCESS_KEY: "{{ env_var('AWS_SECRET_ACCESS_KEY') }}"
AWS_SESSION_TOKEN: session_token (Used to generate Redshift credentials)
AWS_REGION: region
AZURE_CLIENT_ID: "{{ env_var('AZURE_CLIENT_ID') }}"
AZURE_CLIENT_SECRET: "{{ env_var('AZURE_CLIENT_SECRET') }}"
AZURE_STORAGE_ACCOUNT_NAME: "{{ env_var('AZURE_STORAGE_ACCOUNT_NAME') }}"
AZURE_TENANT_ID: "{{ env_var('AZURE_TENANT_ID') }}"
GOOGLE_SERVICE_ACC_KEY:
type: service_account
project_id: project-id
private_key_id: key-id
private_key:
"-----BEGIN PRIVATE KEY-----\nyour_private_key\n-----END_PRIVATE_KEY"
client_email: your_service_account_email
auth_uri: "https://accounts.google.com/o/oauth2/auth"
token_uri: "https://accounts.google.com/o/oauth2/token"
auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs"
client_x509_cert_url: "https://www.googleapis.com/robot/v1/metadata/x509/your_service_account_email"
GOOGLE_SERVICE_ACC_KEY_FILEPATH: "/path/to/your/service/account/key.json"
MYSQL_DATABASE: database
MYSQL_HOST: host
MYSQL_PASSWORD: password
MYSQL_PORT: 3306
MYSQL_USER: root
POSTGRES_DBNAME: postgres
POSTGRES_USER: username
POSTGRES_PASSWORD: password
POSTGRES_HOST: hostname
POSTGRES_PORT: 5432
REDSHIFT_DBNAME: redshift_db_name
REDSHIFT_HOST: redshift_cluster_id.identifier.region.redshift.amazonaws.com
REDSHIFT_PORT: 5439
REDSHIFT_TEMP_CRED_USER: temp_username
REDSHIFT_TEMP_CRED_PASSWORD: temp_password
REDSHIFT_DBUSER: redshift_db_user
REDSHIFT_CLUSTER_ID: redshift_cluster_id
REDSHIFT_IAM_PROFILE: default
SNOWFLAKE_USER: username
SNOWFLAKE_PASSWORD: password
SNOWFLAKE_ACCOUNT: account_id.region
SNOWFLAKE_DEFAULT_WH: default_warehouse
SNOWFLAKE_DEFAULT_DB: optional_default_database
SNOWFLAKE_DEFAULT_SCHEMA: optional_default_schema