Pub/Sub to BigQuery: Ingesta de Streaming sin Servidores con Terraform

terraform {
  required_version = ">= 1.6.0"

  required_providers {
    google = {
      source  = "hashicorp/google"
      version = ">= 7.45.0"
    }
  }
}

variable "project_id" {
  type        = string
  description = "Google Cloud project ID"
}

variable "region" {
  type        = string
  default     = "europe-southwest1"
}

provider "google" {
  project = var.project_id
  region  = var.region
}

resource "google_project_service" "pubsub" {
  service            = "pubsub.googleapis.com"
  disable_on_destroy = false
}

resource "google_project_service" "bigquery" {
  service            = "bigquery.googleapis.com"
  disable_on_destroy = false
}

resource "google_pubsub_topic" "events" {
  name = "analytics-events"

  depends_on = [
    google_project_service.pubsub
  ]
}

resource "google_pubsub_topic" "dead_letter" {
  name = "analytics-events-dlq"

  depends_on = [
    google_project_service.pubsub
  ]
}

resource "google_bigquery_dataset" "analytics" {
  dataset_id = "streaming_analytics"
  location   = "EU"

  depends_on = [
    google_project_service.bigquery
  ]
}

resource "google_bigquery_table" "events" {
  dataset_id = google_bigquery_dataset.analytics.dataset_id
  table_id   = "events"

  deletion_protection = true

  schema = jsonencode([
    {
      name = "event_id"
      type = "STRING"
      mode = "REQUIRED"
    },
    {
      name = "event_type"
      type = "STRING"
      mode = "NULLABLE"
    },
    {
      name = "customer_id"
      type = "STRING"
      mode = "NULLABLE"
    },
    {
      name = "event_ts"
      type = "TIMESTAMP"
      mode = "NULLABLE"
    },
    {
      name = "payload"
      type = "JSON"
      mode = "NULLABLE"
    }
  ])
}

data "google_project" "current" {}

resource "google_project_iam_member" "pubsub_bigquery_writer" {
  project = var.project_id
  role    = "roles/bigquery.dataEditor"

  member = format(
    "serviceAccount:service-%s@gcp-sa-pubsub.iam.gserviceaccount.com",
    data.google_project.current.number
  )
}

resource "google_pubsub_subscription" "bigquery" {
  name  = "analytics-events-to-bigquery"
  topic = google_pubsub_topic.events.id

  bigquery_config {
    table         = "${var.project_id}.${google_bigquery_dataset.analytics.dataset_id}.${google_bigquery_table.events.table_id}"
    write_metadata = true
    use_table_schema = true
  }

  dead_letter_policy {
    dead_letter_topic     = google_pubsub_topic.dead_letter.id
    max_delivery_attempts = 10
  }

  depends_on = [
    google_project_iam_member.pubsub_bigquery_writer
  ]
}

El bloque bigquery_config es la pieza clave: convierte la suscripción en una ruta de entrega directa hacia BigQuery. Terraform expone opciones como use_table_schema, use_topic_schema, write_metadata, drop_unknown_fields y una cuenta de servicio específica.