Enable logging in Amazon ElasticSearch - Cloudformation

If you use cloudformation and is facing difficulties while setting up logging in for ElasticSearch, this blog will help you

ElasticSearch cloudformation

Following is a sample cloudformation template for creating ElasticSearch. It is a part of a nested stack and hence some parameter input validation and things like that are missing but for the scope of this blog, this will suffice.

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Elastic Search",
  "Parameters": {
    "Environment": {
      "Type": "String"
    },
    "ProjectName": {
      "Type": "String"
    },
    "CFVPCSubnetES1": {
      "Type": "String"
    },
    "CFVPCSubnetES2": {
      "Type": "String"
    },
    "CFVPCSGES": {
      "Type": "String"
    },
    "ESUserpassword": {
      "NoEcho": true,
      "Type": "String"
    },
    "ESStorage": {
      "Type": "String"
    }
  },
  "Conditions": {
    "ProdCondition": {
      "Fn::Equals": [
        {
          "Ref": "Environment"
        },
        "prod"
      ]
    }
  },
  "Resources": {
    "CFES": {
      "Type": "AWS::Elasticsearch::Domain",
      "Properties": {
        "AccessPolicies": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "AWS": "*"
              },
              "Action": "es:*",
              "Resource": {
                "Fn::Sub": "arn:aws:es:ap-northeast-1:337609275973:domain/${ProjectName}-${Environment}-es/*"
              }
            }
          ]
        },
        "AdvancedSecurityOptions": {
          "Enabled": true,
          "InternalUserDatabaseEnabled": true,
          "MasterUserOptions": {
            "MasterUserName": {
              "Fn::Sub": "${ProjectName}-${Environment}esadmin"
            },
            "MasterUserPassword": {
              "Ref": "ESUserpassword"
            }
          }
        },
        "DomainEndpointOptions": {
          "EnforceHTTPS": "true",
          "TLSSecurityPolicy": "Policy-Min-TLS-1-2-2019-07"
        },
        "DomainName": {
          "Fn::Sub": "${ProjectName}-${Environment}-es"
        },
        "EBSOptions": {
          "EBSEnabled": "true",
          "VolumeSize": {
            "Ref": "ESStorage"
          },
          "VolumeType": "gp2"
        },
        "ElasticsearchClusterConfig": {
          "Fn::If": [
            "ProdCondition",
            {
              "DedicatedMasterCount": 3,
              "DedicatedMasterEnabled": "true",
              "DedicatedMasterType": "t3.medium.elasticsearch",
              "InstanceCount": "2",
              "InstanceType": "t3.medium.elasticsearch",
              "ZoneAwarenessConfig": {
                "AvailabilityZoneCount": "2"
              },
              "ZoneAwarenessEnabled": "true"
            },
            {
              "DedicatedMasterEnabled": "false",
              "InstanceCount": "1",
              "InstanceType": "t3.small.elasticsearch",
              "ZoneAwarenessEnabled": "false"
            }
          ]
        },
        "ElasticsearchVersion": "7.8",
        "SnapshotOptions": {
          "AutomatedSnapshotStartHour": "15"
        },
        "EncryptionAtRestOptions": {
          "Enabled": true
        },
        "LogPublishingOptions": {
          "ES_APPLICATION_LOGS": {
            "CloudWatchLogsLogGroupArn": {
              "Fn::GetAtt": [
                "CFCWLogGroupApp",
                "Arn"
              ]
            },
            "Enabled": true
          },
          "AUDIT_LOGS": {
            "CloudWatchLogsLogGroupArn": {
              "Fn::GetAtt": [
                "CFCWLogGroupAudit",
                "Arn"
              ]
            },
            "Enabled": true
          }
        },
        "NodeToNodeEncryptionOptions": {
          "Enabled": true
        },
        "VPCOptions": {
          "SecurityGroupIds": [
            {
              "Ref": "CFVPCSGES"
            }
          ],
          "SubnetIds": {
            "Fn::If": [
              "ProdCondition",
              [
                {
                  "Ref": "CFVPCSubnetES1"
                },
                {
                  "Ref": "CFVPCSubnetES2"
                }
              ],
              [
                {
                  "Ref": "CFVPCSubnetES1"
                }
              ]
            ]
          }
        }
      }
    },
    "CFCWLogGroupAudit": {
      "Type": "AWS::Logs::LogGroup",
      "Properties": {
        "LogGroupName": {
          "Fn::Sub": "/aws/aes/domains/${ProjectName}-${Environment}-es/audit-logs"
        }
      }
    },
    "CFCWLogGroupApp": {
      "Type": "AWS::Logs::LogGroup",
      "Properties": {
        "LogGroupName": {
          "Fn::Sub": "/aws/aes/domains/${ProjectName}-${Environment}-es/application-logs"
        }
      }
    }
  },
  "Outputs": {
    "CFES": {
      "Value": {
        "Ref": "CFES"
      }
    },
    "CFESEndpoint": {
      "Value": {
        "Fn::GetAtt": [
          "CFES",
          "DomainEndpoint"
        ]
      }
    }
  }
}

If you run the above cloudformation for the first time in a region in an account, you will face the following error

The Resource Access Policy specified for the CloudWatch Logs log group /es/es-domain/AppLogs does not grant sufficient permissions for Amazon Elasticsearch Service to create a log stream. Please check the Resource Access Policy. (Service: AWSElasticsearch; Status Code: 400; Error Code: ValidationException; Request ID: 915d5df2-6b0c-432b-ad6f-0257ab32a85f; Proxy: null)

Solution

To fix the above issue, run the following command using AWS CLI and the cloudformation template for elastic search logging will work in the next run.

 aws --region ap-northeast-1 logs put-resource-policy --policy-name elasticsearch-logging --policy-document '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": [ "es.amazonaws.com" ] }, "Action": [ "logs:CreateLogStream", "logs:PutLogEvents", "logs:PutLogEventsBatch" ], "Resource": "arn:aws:logs:*:*:log-group:/aws/aes/domains/*:*" } ] }'

What it does is, it creates a resource policy for cloudwatch logs in the specified region which allows elastic search to write logs to the appropriate log group

{
	"Version": "2012-10-17",
	"Statement": [{
		"Effect": "Allow",
		"Principal": {
			"Service": ["es.amazonaws.com"]
		},
		"Action": ["logs:CreateLogStream", "logs:PutLogEvents", "logs:PutLogEventsBatch"],
		"Resource": "arn:aws:logs:*:*:log-group:/aws/aes/domains/*:*"
	}]
}