Terraform wiping out Azure App Configuration settings even if ignore_changes switch is there

Question

I have three web apps in one app service plan. One app is Front End, another is API and the third one is a Function App. All are in a Linux App Service plan.

I have lots of App Config settings in the three apps. What i am noticing is, if i put :

 lifecycle {
    ignore_changes = [
      "auth_settings",
      "app_settings"
    ]
  }

terraform keeps on re writing the app configs from one of the apps after every apply. If i comment out the app_settings of one of the apps to re deploy the app configs, the other one gets wiped out.

Is this a bug or something in Terraform? I am using AzureRM provider 2.0

The code is shown below :

module "name_app_service_plan" {
  version              = "~> 3.0"
  source               = "contoso.com/names/azurerm"
  providers            = { azurerm = azurerm, random = random }
  resource_environment = var.project.environment.name
  resource_location    = var.location
  resource_name        = var.project.name
}

module "name_app_service_api" {
  version              = "~> 3.0"
  source               = "contoso.com/names/azurerm"
  providers            = { azurerm = azurerm, random = random }
  resource_environment = var.project.environment.name
  resource_location    = module.resourcegroup.resource_group.location
  resource_name        = format("%s-api", var.project.name)
}

module "name_app_service_fe" {
  version              = "~> 3.0"
  source               = "contoso.com/names/azurerm"
  providers            = { azurerm = azurerm, random = random }
  resource_environment = var.project.environment.name
  resource_location    = module.resourcegroup.resource_group.location
  resource_name        = format("%s-fe", var.project.name)
}

module "name_function_app" {
  version              = "~> 3.0"
  source               = "contoso.com/names/azurerm"
  providers            = { azurerm = azurerm, random = random }
  resource_environment = var.project.environment.name
  resource_location    = module.resourcegroup.resource_group.location
  resource_name        = format("%s-01", var.project.name)
}

resource "azurerm_app_service_plan" "default" {
  resource_group_name = module.resourcegroup.resource_group.name
  location            = module.resourcegroup.resource_group.location
  name                = module.name_app_service_plan.location.app_service_plan.name_unique
  kind                = "Linux"
  reserved            = true
  sku {
    tier = "PremiumV3"
    size = "P2v3"
  }
  tags = module.resourcegroup.resource_group.tags
}

API App

resource "azurerm_app_service" "api" {
  location            = module.resourcegroup.resource_group.location
  resource_group_name = module.resourcegroup.resource_group.name
  tags                = local.tags

  app_service_plan_id = azurerm_app_service_plan.default.id
  name                = module.name_app_service_api.location.app_service.name_unique
  identity { type = "SystemAssigned" }
  site_config {
    always_on                 = true
    app_command_line          = ""
    default_documents         = []
    dotnet_framework_version  = "v4.0"
    ftps_state                = "AllAllowed"
    health_check_path         = ""
    http2_enabled             = true
    linux_fx_version          = "DOTNETCORE|3.1"
    local_mysql_enabled       = false
    managed_pipeline_mode     = "Integrated"
    min_tls_version           = "1.2"
    python_version            = "3.4"
    remote_debugging_enabled  = false
    remote_debugging_version  = "VS2019"
    use_32_bit_worker_process = false
    windows_fx_version        = ""
    websockets_enabled        = true
    cors {
      allowed_origins     = [format("https://%s", azurerm_app_service.fe.default_site_hostname)]
      support_credentials = true
    }
  }

  app_settings = {
    "WEBSITE_DNS_SERVER"                    = "168.63.129.16"
    "WEBSITE_VNET_ROUTE_ALL"                = "1"
    "WEBSITE_ENABLE_SYNC_UPDATE_SITE"       = "true"
    "APPINSIGHTS_INSTRUMENTATIONKEY"        = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-default-api-instrumentation-key)", module.key-vault.key_vault.self.name)
    "APPLICATIONINSIGHTS_CONNECTION_STRING" = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-api-connection-string)", module.key-vault.key_vault.self.name)
    "applicationStorage"                    = format("@Microsoft.KeyVault(VaultName=%s;SecretName=StorageAccount-ConnectionString-PrimaryKey)", module.key-vault.key_vault.self.name)
    "frontendappid"                         = format("@Microsoft.KeyVault(VaultName=%s;SecretName=frontendappid)", module.key-vault.key_vault.self.name)
    "webapiappid"                           = format("@Microsoft.KeyVault(VaultName=%s;SecretName=webapiappid)", module.key-vault.key_vault.self.name)
    "clientsecret"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=Webapp-API-Secret)", module.key-vault.key_vault.self.name)
    "webapiappuri"                          = format("https://contoso.onmicrosoft.com/api-%s-%s", var.project.name, var.project.environment.name)
    "functionappid"                         = format("@Microsoft.KeyVault(VaultName=%s;SecretName=functionappid)", module.key-vault.key_vault.self.name)
    "functionappuri"                        = format("https://contoso.onmicrosoft.com/func-%s-%s", var.project.name, var.project.environment.name)
    "funcappsecret"                         = format("@Microsoft.KeyVault(VaultName=%s;SecretName=Function-App-Secret)", module.key-vault.key_vault.self.name)
    "frontendhost"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=frontendhost)", module.key-vault.key_vault.self.name)
    "functionhost"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=functionhost)", module.key-vault.key_vault.self.name)
    "webapihost"                            = format("@Microsoft.KeyVault(VaultName=%s;SecretName=webapihost)", module.key-vault.key_vault.self.name)
    "cosmosIntegrationConnection"           = format("@Microsoft.KeyVault(VaultName=%s;SecretName=CosmosDB-PrimaryKey-ConnectionString)", module.key-vault.key_vault.self.name)
    "cosmosIntegrationContainer"            = "cosmosdb_container"
    "cosmosIntegrationDatabase"             = "cosmosdb_db"
    "tokenauthority"                        = format("https://login.microsoftonline.com/%s", data.azurerm_client_config.default.tenant_id)
  }

  lifecycle {
    ignore_changes = [
      auth_settings,
      app_settings
    ]
  }
}

VNET Integration for api app

resource "azurerm_app_service_virtual_network_swift_connection" "api" {
  app_service_id = azurerm_app_service.api.id
  subnet_id      = module.virtualnetwork["centralus"].virtual_network.subnets["webapp"].id
}

Pvt Endpoint for API App

module "privateendpoint_api" {
  # registry
  version = "~> 10.0.0"
  source  = "contoso.com/virtual-network/azurerm//modules/privateendpoint"
  # metas
  providers = { azurerm = azurerm, azurerm.hub = azurerm.hub, random = random }
  # arguments
  hub_resource_group_name               = var.project.hub.resourcegroup.name
  resource_group_name                   = module.resourcegroup.resource_group.name
  private_endpoint_location             = module.resourcegroup.resource_group.location
  private_endpoint_environment          = var.project.environment.name
  private_endpoint_name                 = format("api-webapp-%s", var.project.name)
  private_endpoint_resource_type        = "appservice"
  private_endpoint_resource_subresource = "sites"
  private_endpoint_subnet_id            = module.virtualnetwork["centralus"].virtual_network.subnets["general"].id
  private_endpoint_resource_id          = azurerm_app_service.api.id
}

FE Web App

resource "azurerm_app_service" "fe" {
  location            = module.resourcegroup.resource_group.location
  resource_group_name = module.resourcegroup.resource_group.name
  tags                = local.tags

  app_service_plan_id = azurerm_app_service_plan.default.id
  name                = module.name_app_service_fe.location.app_service.name_unique
  identity { type = "SystemAssigned" }
  auth_settings {
    enabled                        = true
    default_provider               = "AzureActiveDirectory"
    issuer                         = format("https://sts.windows.net/%s/", data.azurerm_client_config.default.tenant_id)
    runtime_version                = "~1"
    token_store_enabled            = true
    unauthenticated_client_action  = "RedirectToLoginPage"
    allowed_external_redirect_urls = module.application_webapp_fe.application.self.reply_urls
    additional_login_params = {
      "response_type" = "code id_token",
      "resource"      = module.application_webapp_fe.application.self.application_id
    }
    active_directory {
      client_id         = module.application_webapp_fe.application.self.application_id
      client_secret     = module.application_webapp_fe.service_principal.secret.value
      allowed_audiences = []
    }
  }

  site_config {
    always_on                = true
    app_command_line         = ""
    default_documents        = []
    dotnet_framework_version = "v4.0"
    ftps_state               = "Disabled"
    health_check_path        = ""
    http2_enabled            = true
    linux_fx_version         = "STATICSITE|1.0"
    local_mysql_enabled      = false
    managed_pipeline_mode    = "Integrated"
    min_tls_version          = "1.2"
    #pre_warmed_instance_count = 0
    python_version            = "3.4"
    remote_debugging_enabled  = false
    remote_debugging_version  = "VS2019"
    use_32_bit_worker_process = false
    websockets_enabled        = false
    windows_fx_version        = ""
    cors {
      allowed_origins     = []
      support_credentials = false
    }
  }

  app_settings = {
    "WEBSITE_DNS_SERVER"                       = "168.63.129.16"
    "WEBSITE_VNET_ROUTE_ALL"                   = "1"
    "MICROSOFT_PROVIDER_AUTHENTICATION_SECRET" = format("@Microsoft.KeyVault(VaultName=%s;SecretName=Webapp-FE-Secret)", module.key-vault.key_vault.self.name)
    "APPINSIGHTS_INSTRUMENTATIONKEY"           = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-default-fe-instrumentation-key)", module.key-vault.key_vault.self.name)
    "APPLICATIONINSIGHTS_CONNECTION_STRING"    = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-fe-connection-string)", module.key-vault.key_vault.self.name)
    "webapiappid"                              = format("@Microsoft.KeyVault(VaultName=%s;SecretName=webapiappid)", module.key-vault.key_vault.self.name)
    "webapiappsecret"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=Webapp-API-Secret)", module.key-vault.key_vault.self.name)
  }

  lifecycle {
    ignore_changes = [
      auth_settings,
      app_settings
    ]
  }
}

VNET Integration for fe app

resource "azurerm_app_service_virtual_network_swift_connection" "fe" {
  app_service_id = azurerm_app_service.fe.id
  subnet_id      = module.virtualnetwork["centralus"].virtual_network.subnets["webapp"].id
}

Pvt Endpoint for Fe App

module "privateendpoint_fe" {
  # registry
  version = "~> 10.0.0"
  source  = "contoso.com/virtual-network/azurerm//modules/privateendpoint"
  # metas
  providers = { azurerm = azurerm, azurerm.hub = azurerm.hub, random = random }
  # arguments
  hub_resource_group_name               = var.project.hub.resourcegroup.name
  resource_group_name                   = module.resourcegroup.resource_group.name
  private_endpoint_location             = module.resourcegroup.resource_group.location
  private_endpoint_environment          = var.project.environment.name
  private_endpoint_name                 = format("fe-webapp-%s", var.project.name)
  private_endpoint_resource_type        = "appservice"
  private_endpoint_resource_subresource = "sites"
  private_endpoint_subnet_id            = module.virtualnetwork["centralus"].virtual_network.subnets["general"].id
  private_endpoint_resource_id          = azurerm_app_service.fe.id
}

Function App

resource "azurerm_function_app" "default" {
  location            = module.resourcegroup.resource_group.location
  resource_group_name = module.resourcegroup.resource_group.name
  tags                = local.tags

  app_service_plan_id = azurerm_app_service_plan.default.id
  name                = module.name_function_app.location.function_app.name_unique
  version             = "~3"
  identity { type = "SystemAssigned" }
  os_type                    = "linux"
  storage_account_name       = module.storageaccount.storage_account.self.name
  storage_account_access_key = module.storageaccount.storage_account.self.primary_access_key

  auth_settings {
    enabled                       = true
    default_provider              = "AzureActiveDirectory"
    issuer                        = format("https://login.microsoftonline.com/%s/", data.azurerm_client_config.default.tenant_id)
    runtime_version               = "~1"
    token_store_enabled           = true
    unauthenticated_client_action = "RedirectToLoginPage"
    active_directory {
      client_id         = module.application_func_01.application.self.application_id
      client_secret     = module.application_func_01.service_principal.secret.value
      allowed_audiences = module.application_webapp_api.application.self.identifier_uris
    }
  }

  site_config {
    always_on                 = true
    ftps_state                = "Disabled"
    health_check_path         = ""
    http2_enabled             = true
    linux_fx_version          = "DOCKER|mcr.microsoft.com/azure-functions/dotnet:3.0-dotnet3-appservice"
    min_tls_version           = "1.2"
    pre_warmed_instance_count = 0
    use_32_bit_worker_process = false
    websockets_enabled        = false
    cors {
      allowed_origins     = []
      support_credentials = false
    }
  }

  app_settings = {
    "WEBSITE_DNS_SERVER"                    = "168.63.129.16"
    "WEBSITE_VNET_ROUTE_ALL"                = "1"
    "WEBSITE_ENABLE_SYNC_UPDATE_SITE"       = "true"
    "WEBSITE_RUN_FROM_PACKAGE"              = "1"
    "WEBSITES_ENABLE_APP_SERVICE_STORAGE"   = "true"
    "APPINSIGHTS_INSTRUMENTATIONKEY"        = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-default-func-instrumentation-key)", module.key-vault.key_vault.self.name)
    "APPLICATIONINSIGHTS_CONNECTION_STRING" = format("@Microsoft.KeyVault(VaultName=%s;SecretName=appi-func-connection-string)", module.key-vault.key_vault.self.name)
    "AzureWebJobsStorage"                   = format("@Microsoft.KeyVault(VaultName=%s;SecretName=StorageAccount-FunctionApp-ConnectionString-PrimaryKey)", module.key-vault.key_vault.self.name)
    "AzureWebJobsDashboard"                 = format("@Microsoft.KeyVault(VaultName=%s;SecretName=StorageAccount-FunctionApp-ConnectionString-PrimaryKey)", module.key-vault.key_vault.self.name)
    "cosmosIntegrationConnection"           = format("@Microsoft.KeyVault(VaultName=%s;SecretName=CosmosDB-PrimaryKey-ConnectionString)", module.key-vault.key_vault.self.name)
    "cosmosIntegrationContainer"            = "cosmosdb_container"
    "cosmosIntegrationDatabase"             = "cosmosdb_db"
    "functionappid"                         = format("@Microsoft.KeyVault(VaultName=%s;SecretName=functionappid)", module.key-vault.key_vault.self.name)
    "clientsecret"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=Function-App-Secret)", module.key-vault.key_vault.self.name)
    "webapihost"                            = format("@Microsoft.KeyVault(VaultName=%s;SecretName=webapihost)", module.key-vault.key_vault.self.name)
    "webapiappuri"                          = format("https://contoso.onmicrosoft.com/api-%s-%s", var.project.name, var.project.environment.name)
    "functionappuri"                        = format("https://contoso.onmicrosoft.com/func-%s-%s", var.project.name, var.project.environment.name)
    "functionhost"                          = format("@Microsoft.KeyVault(VaultName=%s;SecretName=functionhost)", module.key-vault.key_vault.self.name)
    "applicationStorage"                    = format("@Microsoft.KeyVault(VaultName=%s;SecretName=StorageAccount-FunctionApp-ConnectionString-PrimaryKey)", module.key-vault.key_vault.self.name)
    "tokenauthority"                        = format("https://login.microsoftonline.com/%s", data.azurerm_client_config.default.tenant_id)
    "FUNCTIONS_EXTENSION_VERSION"           = "~3"

  }

  lifecycle {
    ignore_changes = [
      auth_settings,
      app_settings
    ]
  }
}

VNET Integration for function app

resource "azurerm_app_service_virtual_network_swift_connection" "function-app" {
  app_service_id = azurerm_function_app.default.id
  subnet_id      = module.virtualnetwork["centralus"].virtual_network.subnets["webapp"].id
}

Pvt Endpoint for Func App

module "privateendpoint_func" {
  # registry
  version = "~> 10.0.0"
  source  = "contoso.com/virtual-network/azurerm//modules/privateendpoint"
  # metas
  providers = { azurerm = azurerm, azurerm.hub = azurerm.hub, random = random }
  # arguments
  hub_resource_group_name               = var.project.hub.resourcegroup.name
  resource_group_name                   = module.resourcegroup.resource_group.name
  private_endpoint_location             = module.resourcegroup.resource_group.location
  private_endpoint_environment          = var.project.environment.name
  private_endpoint_name                 = format("func-%s", var.project.name)
  private_endpoint_resource_type        = "appservice"
  private_endpoint_resource_subresource = "sites"
  private_endpoint_subnet_id            = module.virtualnetwork["centralus"].virtual_network.subnets["general"].id
  private_endpoint_resource_id          = azurerm_function_app.default.id
}

What i am noticing is Terraform only respects the ignore_changes value for the Function App. For the API and FE app, if i comment out ignore_changes for the app_settings for say FE, Terraform also wipes out the app_settings for API app and only keeps the VNET_ROUTE_ALL setting. All the other settings get wiped out.

And same thing happens if i comment out the app_settings in ignore_changes for the API app and uncomment the app_settings of the FE App.

Answer 1

As mentioned in the comments a look at output of tf plan is important. But given the information in your question I can think of three potential culprits:

• If a dependent resource is recreated or changed the app might have to be recreated, which leads to wiping app_settings (at least the settings added through the portal).

• You are using a remote module with a pessimistic constraint "~> 3.0", so if there is a new version (say 3.1) and you are deploying on a remote agents without cache or locally with an -upgrade flag Terraform will redeploy the module in order to get the latest version.

• This should not be the problem but it's worth mentioning: ignoring specific changes in an object with other objects inside has been problematic in the past.

• There seems to have been specific problems for azurerm 2.0, Terraform would recreate the entire app when app-settings changed, which would then make the ignore_changes redundant, this should have been fixed in version 2.1

That said, atm the resource is deprecated; so an upgrade seems the easiest fix. See: deprecated & latest

For anyone trying to reproduce this issue; here is a simple code example using the azurerm provide "2.48.0" and tf version 1.3.9. This example shows no problems with ignoring the changes made either in the portal or in the Terraform code.

Folder structure:

main.tf

# locals
locals {
  resource_environment = "dev"
  resource_location    = "westeurope"
  resource_name        = "stack" # name of the resource group

  resource_name_app      = "stack12345678" # needs to be unique
  resource_name_function = "stack87654321" # needs to be unique
  tags = {
    environment = local.resource_environment
    project     = local.resource_name
  }
}

terraform {
  required_version = ">= 0.13.0"
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "= 2.48.0"
    }
    random = {
      source  = "hashicorp/random"
      version = ">= 2.0.0"
    }
  }
}

provider "azurerm" {
  features {}
}

resource "azurerm_resource_group" "stack_test" {
  name     = local.resource_name
  location = local.resource_location
  tags     = local.tags
}

module "test_app_service2" {
  source    = "./module_app_service"
  providers = { azurerm = azurerm, random = random }
  #resource_environment = local.resource_environment
  resource_location   = local.resource_location
  resource_group_name = local.resource_name
  resource_name       = local.resource_name_app
  tags                = local.tags

  depends_on = [
    azurerm_resource_group.stack_test
  ]
}

module "test_function_app1" {
  source    = "./module_function_app"
  providers = { azurerm = azurerm, random = random }
  #resource_environment = local.resource_environment
  resource_location   = local.resource_location
  resource_group_name = local.resource_name
  resource_name       = local.resource_name_function
  tags                = local.tags

  depends_on = [
    azurerm_resource_group.stack_test
  ]
}

module_function.tf

variable "resource_location" {}
variable "resource_group_name" {}
variable resource_name {}
variable "tags" {}

resource "azurerm_app_service_plan" "default" {
  location            = var.resource_location
  resource_group_name = var.resource_group_name
  tags                = var.tags

  name     = "api-appserviceplan-pro"
  kind     = "Linux"
  reserved = true

  sku {
    tier = "Standard"
    size = "S1"
  }
}

resource "azurerm_storage_account" "example" {
  name                     = var.resource_name
  resource_group_name      = var.resource_group_name
  location                 = var.resource_location
  account_tier             = "Standard"
  account_replication_type = "LRS"
}

resource "azurerm_function_app" "default" {
  location            = var.resource_location
  resource_group_name = var.resource_group_name
  tags                = var.tags

  app_service_plan_id = azurerm_app_service_plan.default.id
  name                = var.resource_name
  version             = "~3"

  identity {
    type = "SystemAssigned"
  }

  os_type                    = "linux"
  storage_account_name       = azurerm_storage_account.example.name
  storage_account_access_key = azurerm_storage_account.example.primary_access_key

  auth_settings {
    enabled                       = true
  }

  site_config {
    always_on                 = true
    ftps_state                = "Disabled"
    health_check_path         = ""
    http2_enabled             = true
    linux_fx_version          = "DOCKER|mcr.microsoft.com/azure-functions/dotnet:3.0-dotnet3-appservice"
    min_tls_version           = "1.2"
    pre_warmed_instance_count = 0
    use_32_bit_worker_process = false
    websockets_enabled        = false
  }

  app_settings = {
    something = "new"
    name_service_plan =  azurerm_app_service_plan.default.name
    new = "this"}

  lifecycle {
    ignore_changes = [
      auth_settings,
      app_settings
    ]
  }
}

module_app_service.tf

variable "resource_location" {}
variable "resource_group_name" {}
variable resource_name {}
variable "tags" {}

resource "azurerm_app_service_plan" "default" {
  location            = var.resource_location
  resource_group_name = var.resource_group_name
  tags                = var.tags 

  name     = "api-appserviceplan-pro"
  kind     = "Linux"
  reserved = true

  sku {
    tier = "Standard"
    size = "S1"
  }

}

resource "azurerm_app_service" "api" {
  location            = var.resource_location
  resource_group_name = var.resource_group_name
  tags                = var.tags

  app_service_plan_id = azurerm_app_service_plan.default.id
  name                = var.resource_name
  identity { type = "SystemAssigned" }
  site_config {
    always_on                 = true
    app_command_line          = ""
    default_documents         = []
    dotnet_framework_version  = "v4.0"
    ftps_state                = "AllAllowed"
    health_check_path         = ""
    http2_enabled             = true
    linux_fx_version          = "DOTNETCORE|3.1"
    local_mysql_enabled       = false
    managed_pipeline_mode     = "Integrated"
    min_tls_version           = "1.2"
    python_version            = "3.4"
    remote_debugging_enabled  = false
    remote_debugging_version  = "VS2019"
    use_32_bit_worker_process = false
    windows_fx_version        = ""
    websockets_enabled        = true

    cors {
      allowed_origins     = []
      support_credentials = false
    }
  }

  app_settings = {
    something = "some thing"
    name_service_plan =  azurerm_app_service_plan.default.name
    new = "this"}

  lifecycle {
    ignore_changes = [
      auth_settings,
      app_settings
    ]
  }
}

see also:

Terraform, "ignore_changes" and sub-blocks

How to ignore change of an attribute in block

https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes