Huston, we got a problem!!! 🙂 Random our metaflow service stop working, today around 9 UTC we think its was something we deployed but we got this error

2023/12/12 13:03:30 goose run: failed to connect to `host=metaflow-metaflow-579p5yo1.[key].<http://eu-west-1.rds.amazonaws.com|eu-west-1.rds.amazonaws.com> user=metaflow database=metaflow`: server error (FATAL: no pg_hba.conf entry for host "10.10.2.137", user "metaflow", database "metaflow", no encryption (SQLSTATE 28000))

We have trying anything, even destory our stack wtih Terraform, upgrade to lates version but still have this issue, we are image version: netflixoss/metaflow_metadata_service:v2.4.5, before we use latest but get stuck on a 2year old image so now we custom change that image. So even after er full redeploy from ground we still se this problem, the RDS database are Postgres 16.1 before 15.4 with same issue So anyhow have this problem before and whats can be changed our terraform setup look like this and all versions are updated to the latest terraform modules.

# Random suffix for this deployment
resource "random_string" "suffix" {
  length  = 8
  special = false
  upper   = false
}

locals {
  resource_prefix = "metaflow"
  resource_suffix = random_string.suffix.result
}

module "metaflow" {
  source  = "outerbounds/metaflow/aws"
  version = "0.10.1"

  resource_prefix = local.resource_prefix
  resource_suffix = local.resource_suffix

  ui_certificate_arn        = "arn:aws:acm:eu-west-1:<acc-id>:certificate/<uuid>"
  ui_static_container_image = "<acc-id>.<http://dkr.ecr.eu-west-1.amazonaws.com/metaflow-ui-frontend:latest|dkr.ecr.eu-west-1.amazonaws.com/metaflow-ui-frontend:latest>"
  ui_allow_list             = ["range-list"]
  db_instance_type          = "db.m6g.large"
  db_engine_version          = "16.1"
  
  metadata_service_container_image = "netflixoss/metaflow_metadata_service:v2.4.5"

  enable_step_functions = false
  subnet1_id            = data.aws_subnets.public_subnets.ids[0]
  subnet2_id            = data.aws_subnets.public_subnets.ids[1]
  vpc_cidr_blocks       = ["10.10.0.0/16"]
  vpc_id                = data.aws_vpc.vpc.id
  with_public_ip        = true
  force_destroy_s3_bucket = true
  
  tags = {
    "managedBy" = "terraform"
  }
}

# export all outputs from metaflow modules
output "metaflow" {
  value = module.metaflow
}

# The module will generate a Metaflow config in JSON format, write it to a file
resource "local_file" "metaflow_config" {
  content  = module.metaflow.metaflow_profile_json
  filename = "./metaflow_profile.json"
}