diff --git a/README.md b/README.md index 27ab6b5..d3c1de8 100644 --- a/README.md +++ b/README.md @@ -96,10 +96,10 @@ go build -o om ./cmd/om # Or scan cloud / cluster inventory export AWS_REGION=us-east-1 ./om scan aws -# On an EC2 instance in that account, as root, after the API scan: -# sudo -E ./om scan workload aws -# sudo -E ./om scan workload image aws -# On an Azure VM or a GCE instance in that region, as root: +# On the scanner worker, as root, after the API scan. +# Production grants are in deploy/connect. +# sudo -E ./om scan workload aws --snapshot-role "$ROLE_ARN" --external-id "$EXTERNAL_ID" --kms-key "$SCANNER_KEY" +# sudo -E ./om scan workload image aws --snapshot-role "$ROLE_ARN" --external-id "$EXTERNAL_ID" --kms-key "$SCANNER_KEY" # sudo -E ./om scan workload azure # sudo -E ./om scan workload image azure # sudo -E ./om scan workload gcp @@ -146,14 +146,14 @@ An API token is created only when an admin asks for one, from the console or wit ```bash ./om scan demo # sample environment, no credentials ./om scan aws -./om scan workload aws # root on an EC2 worker; Linux root-volume packages -./om scan workload image aws # root on an EC2 worker; AMI packages, applied to instances +./om scan workload aws --snapshot-role "$ROLE_ARN" --external-id "$EXTERNAL_ID" --kms-key "$SCANNER_KEY" +./om scan workload image aws --snapshot-role "$ROLE_ARN" --external-id "$EXTERNAL_ID" --kms-key "$SCANNER_KEY" ./om scan azure # requires AZURE_SUBSCRIPTION_ID + az login -./om scan workload azure # root on an Azure VM; Linux OS-disk packages -./om scan workload image azure # root on an Azure VM; boot-image packages, applied to VMs +./om scan workload azure # scanner VM; snapshots the production subscription +./om scan workload image azure # scanner VM; boot-image packages, applied to VMs ./om scan gcp # requires GCP_PROJECT_ID + ADC -./om scan workload gcp # root on a GCE instance; Linux boot-disk packages -./om scan workload image gcp # root on a GCE instance; image packages, applied to instances +./om scan workload gcp # scanner VM; snapshots the production project +./om scan workload image gcp # scanner VM; image packages, applied to instances ./om scan k8s # requires kubeconfig or in-cluster credentials ./om scan plugin -- ./my-collector ``` diff --git a/collectors/README.md b/collectors/README.md index a2de956..6434f1f 100644 --- a/collectors/README.md +++ b/collectors/README.md @@ -20,7 +20,7 @@ Cloud and platform ingestion plugins. Each collector normalizes provider APIs in The **demo** collector loads a fixed environment (internet-exposed web tier, production database, public/private buckets, admin vs app identities, public Kubernetes service). Use it to exercise CSPM packs without cloud credentials. -The **AWS** collector emits properties the CIS pack matches on: `open_ingress`, `imdsv2`, `public_ip`, S3 `encryption` / `versioning` / `public_access_block`, and IAM user `mfa` / `unused_access_keys`. It also records RDS and Aurora instances as datastores. S3 bucket tags and the RDS `TagList` copy `sensitivity` or `data-class` onto the datastore. `sensitivity` wins when both are set. The same scan reads CloudTrail management events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. `om scan aws` does not read instance disks. `om scan workload aws` inventories packages on Linux root volumes. `om scan workload image aws` reads each boot image once and applies those packages to instances launched from it. +The **AWS** collector emits properties the CIS pack matches on: `open_ingress`, `imdsv2`, `public_ip`, S3 `encryption` / `versioning` / `public_access_block`, and IAM user `mfa` / `unused_access_keys`. It also records RDS and Aurora instances as datastores. S3 bucket tags and the RDS `TagList` copy `sensitivity` or `data-class` onto the datastore. `sensitivity` wins when both are set. The same scan reads CloudTrail management events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. `om scan aws` does not read instance disks. `om scan workload aws` inventories packages on Linux root volumes from a worker in a scanner account. `om scan workload image aws` reads each boot image once and applies those packages to instances launched from it. The production role is in `deploy/connect`. The **Azure** collector records logical SQL servers. `om scan azure` and `om scan gcp` do not read OS disks. `om scan workload azure` and `om scan workload gcp` inventory packages on those disks. `om scan workload image azure` and `om scan workload image gcp` read each boot image once. The same scan reads administrative Activity Log events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. Entra ID sign-in logs are not part of this scan. The **GCP** collector records Cloud SQL instances. The same scan reads Admin Activity audit logs from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. Data Access logs are not part of this scan. A workload gets `CAN_ACCESS` only when a security group, firewall, or VPC path allows it. Azure copies the crown-jewel mark from storage-account and SQL-server tags. GCP copies it from a bucket label or a Cloud SQL user label. The keys are `sensitivity` and `data-class`. A plugin may set `sensitivity` on a datastore directly. diff --git a/deploy/connect/README.md b/deploy/connect/README.md new file mode 100644 index 0000000..a23b131 --- /dev/null +++ b/deploy/connect/README.md @@ -0,0 +1,23 @@ +# Connect a production account + +The workload scan mounts disks only in a scanner account, subscription, or project that you run. Each production account grants a role that can snapshot and share. It does not grant permission to attach a disk or start an instance. + +Apply the scanner stack once, then apply the production stack in every account you want to read. Terraform and OpenTofu use the same files. AWS also has CloudFormation and a shell script. Azure has an `az` script. GCP has a `gcloud` script. + +| Cloud | Scanner | Production | +| --- | --- | --- | +| AWS | [`aws/scanner`](aws/scanner) | [`aws/production`](aws/production) | +| Azure | [`azure/scanner`](azure/scanner) | [`azure/production`](azure/production) | +| GCP | [`gcp/scanner`](gcp/scanner) | [`gcp/production`](gcp/production) | + +Create the scanner role first. Pass its ARN, principal id, or service account into the production stack, with an external id you choose and store. The production key policy has to allow that principal to decrypt root-volume keys. The snippet is [`aws/kms-key-policy.json`](aws/kms-key-policy.json). Add the same principal to each Azure disk-encryption key vault and each GCP key that encrypts a boot disk. + +Then, on the scanner worker, as root: + +```bash +sudo -E ./om scan workload aws --snapshot-role "$ROLE_ARN" --external-id "$EXTERNAL_ID" --kms-key "$SCANNER_KEY" +sudo -E ./om scan workload azure +sudo -E ./om scan workload gcp +``` + +`AZURE_SUBSCRIPTION_ID` and `GCP_PROJECT_ID` are the production subscription and project. The worker's own subscription and project come from the instance metadata. AWS snapshots are copied into the scanner account with `--kms-key` when the source volume is encrypted. diff --git a/deploy/connect/aws/kms-key-policy.json b/deploy/connect/aws/kms-key-policy.json new file mode 100644 index 0000000..3204a17 --- /dev/null +++ b/deploy/connect/aws/kms-key-policy.json @@ -0,0 +1,20 @@ +{ + "Sid": "AllowOpenSourceOMScanner", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::SCANNER_ACCOUNT_ID:role/opensourceom-worker" + }, + "Action": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:CreateGrant", + "kms:ReEncryptFrom", + "kms:ReEncryptTo" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "kms:ViaService": "ec2.REGION.amazonaws.com" + } + } +} diff --git a/deploy/connect/aws/production/apply.sh b/deploy/connect/aws/production/apply.sh new file mode 100644 index 0000000..87109a0 --- /dev/null +++ b/deploy/connect/aws/production/apply.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Apply the production snapshot role with the AWS CLI. +# Terraform and OpenTofu use main.tf. CloudFormation uses cloudformation.yaml. +set -eu + +: "${SCANNER_ROLE_ARN:?set SCANNER_ROLE_ARN}" +: "${SCANNER_ACCOUNT_ID:?set SCANNER_ACCOUNT_ID}" +: "${EXTERNAL_ID:?set EXTERNAL_ID}" + +trust=$(cat </dev/null 2>&1; then + aws iam update-assume-role-policy --role-name opensourceom-snapshot --policy-document "$trust" +else + aws iam create-role --role-name opensourceom-snapshot --assume-role-policy-document "$trust" +fi +aws iam put-role-policy --role-name opensourceom-snapshot --policy-name snapshot --policy-document "$policy" +aws iam get-role --role-name opensourceom-snapshot --query Role.Arn --output text diff --git a/deploy/connect/aws/production/cloudformation.yaml b/deploy/connect/aws/production/cloudformation.yaml new file mode 100644 index 0000000..e3610ed --- /dev/null +++ b/deploy/connect/aws/production/cloudformation.yaml @@ -0,0 +1,87 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Grant the OpenSourceOM scanner account permission to snapshot and share. This stack does not attach a disk. + +Parameters: + ScannerRoleArn: + Type: String + Description: ARN of opensourceom-worker in the scanner account. + ScannerAccountId: + Type: String + AllowedPattern: "[0-9]{12}" + ExternalId: + Type: String + NoEcho: true + +Resources: + SnapshotRole: + Type: AWS::IAM::Role + Properties: + RoleName: opensourceom-snapshot + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: !Ref ScannerRoleArn + Action: sts:AssumeRole + Condition: + StringEquals: + sts:ExternalId: !Ref ExternalId + Policies: + - PolicyName: snapshot + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: Read + Effect: Allow + Action: + - ec2:DescribeInstances + - ec2:DescribeImages + - ec2:DescribeSnapshots + - ec2:DescribeVolumes + Resource: "*" + - Sid: CreateTaggedSnapshot + Effect: Allow + Action: + - ec2:CreateSnapshot + - ec2:CreateTags + Resource: + - arn:aws:ec2:*:*:volume/* + - arn:aws:ec2:*::snapshot/* + Condition: + StringEquals: + aws:RequestTag/opensourceom:scan: workload + - Sid: ShareWithScanner + Effect: Allow + Action: ec2:ModifySnapshotAttribute + Resource: arn:aws:ec2:*::snapshot/* + Condition: + StringEquals: + ec2:Add/userId: !Ref ScannerAccountId + - Sid: Unshare + Effect: Allow + Action: ec2:ModifySnapshotAttribute + Resource: arn:aws:ec2:*::snapshot/* + Condition: + StringEquals: + ec2:Remove/userId: !Ref ScannerAccountId + - Sid: DeleteOurSnapshots + Effect: Allow + Action: ec2:DeleteSnapshot + Resource: arn:aws:ec2:*::snapshot/* + Condition: + StringEquals: + ec2:ResourceTag/opensourceom:scan: workload + - Sid: EncryptedVolumes + Effect: Allow + Action: + - kms:CreateGrant + - kms:Decrypt + - kms:DescribeKey + - kms:ReEncryptFrom + - kms:ReEncryptTo + Resource: "*" + +Outputs: + RoleArn: + Value: !GetAtt SnapshotRole.Arn diff --git a/deploy/connect/aws/production/main.tf b/deploy/connect/aws/production/main.tf new file mode 100644 index 0000000..90c00cb --- /dev/null +++ b/deploy/connect/aws/production/main.tf @@ -0,0 +1,119 @@ +variable "scanner_role_arn" { + type = string + description = "ARN of opensourceom-worker in the scanner account." +} + +variable "scanner_account_id" { + type = string + description = "Account id that snapshots are shared with." +} + +variable "external_id" { + type = string + description = "Value the worker passes to sts:AssumeRole." +} + +data "aws_iam_policy_document" "trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [var.scanner_role_arn] + } + condition { + test = "StringEquals" + variable = "sts:ExternalId" + values = [var.external_id] + } + } +} + +data "aws_iam_policy_document" "snapshot" { + statement { + sid = "Read" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeImages", + "ec2:DescribeSnapshots", + "ec2:DescribeVolumes", + ] + resources = ["*"] + } + + statement { + sid = "CreateTaggedSnapshot" + actions = [ + "ec2:CreateSnapshot", + "ec2:CreateTags", + ] + resources = [ + "arn:aws:ec2:*:*:volume/*", + "arn:aws:ec2:*::snapshot/*", + ] + condition { + test = "StringEquals" + variable = "aws:RequestTag/opensourceom:scan" + values = ["workload"] + } + } + + statement { + sid = "ShareWithScanner" + actions = ["ec2:ModifySnapshotAttribute"] + resources = ["arn:aws:ec2:*::snapshot/*"] + condition { + test = "StringEquals" + variable = "ec2:Add/userId" + values = [var.scanner_account_id] + } + } + + statement { + sid = "Unshare" + actions = ["ec2:ModifySnapshotAttribute"] + resources = ["arn:aws:ec2:*::snapshot/*"] + condition { + test = "StringEquals" + variable = "ec2:Remove/userId" + values = [var.scanner_account_id] + } + } + + statement { + sid = "DeleteOurSnapshots" + actions = ["ec2:DeleteSnapshot"] + resources = ["arn:aws:ec2:*::snapshot/*"] + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/opensourceom:scan" + values = ["workload"] + } + } + + statement { + sid = "EncryptedVolumes" + actions = [ + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + ] + resources = ["*"] + } +} + +resource "aws_iam_role" "snapshot" { + name = "opensourceom-snapshot" + assume_role_policy = data.aws_iam_policy_document.trust.json +} + +resource "aws_iam_role_policy" "snapshot" { + name = "snapshot" + role = aws_iam_role.snapshot.id + policy = data.aws_iam_policy_document.snapshot.json +} + +output "role_arn" { + value = aws_iam_role.snapshot.arn +} diff --git a/deploy/connect/aws/scanner/apply.sh b/deploy/connect/aws/scanner/apply.sh new file mode 100644 index 0000000..2a59480 --- /dev/null +++ b/deploy/connect/aws/scanner/apply.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Apply the scanner instance profile with the AWS CLI. +set -eu + +: "${EXTERNAL_ID:?set EXTERNAL_ID}" + +trust='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}' + +policy=$(cat </dev/null 2>&1; then + aws iam update-assume-role-policy --role-name opensourceom-worker --policy-document "$trust" +else + aws iam create-role --role-name opensourceom-worker --assume-role-policy-document "$trust" +fi +aws iam put-role-policy --role-name opensourceom-worker --policy-name worker --policy-document "$policy" +aws iam create-instance-profile --instance-profile-name opensourceom-worker 2>/dev/null || true +aws iam add-role-to-instance-profile --instance-profile-name opensourceom-worker --role-name opensourceom-worker 2>/dev/null || true +aws iam get-role --role-name opensourceom-worker --query Role.Arn --output text diff --git a/deploy/connect/aws/scanner/cloudformation.yaml b/deploy/connect/aws/scanner/cloudformation.yaml new file mode 100644 index 0000000..8700d86 --- /dev/null +++ b/deploy/connect/aws/scanner/cloudformation.yaml @@ -0,0 +1,69 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Instance profile for the OpenSourceOM worker. Launch it in this account and tag the instance opensourceom:worker=true. + +Parameters: + ExternalId: + Type: String + NoEcho: true + +Resources: + WorkerRole: + Type: AWS::IAM::Role + Properties: + RoleName: opensourceom-worker + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: worker + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: CopyAndAttach + Effect: Allow + Action: + - ec2:DescribeInstances + - ec2:DescribeSnapshots + - ec2:DescribeVolumes + - ec2:CopySnapshot + - ec2:CreateVolume + - ec2:CreateTags + - ec2:AttachVolume + - ec2:DetachVolume + - ec2:DeleteVolume + - ec2:DeleteSnapshot + Resource: "*" + - Sid: AssumeProduction + Effect: Allow + Action: sts:AssumeRole + Resource: arn:aws:iam::*:role/opensourceom-snapshot + Condition: + StringEquals: + sts:ExternalId: !Ref ExternalId + - Sid: Keys + Effect: Allow + Action: + - kms:CreateGrant + - kms:Decrypt + - kms:Encrypt + - kms:DescribeKey + - kms:GenerateDataKeyWithoutPlaintext + - kms:ReEncryptFrom + - kms:ReEncryptTo + Resource: "*" + WorkerProfile: + Type: AWS::IAM::InstanceProfile + Properties: + InstanceProfileName: opensourceom-worker + Roles: + - !Ref WorkerRole + +Outputs: + RoleArn: + Value: !GetAtt WorkerRole.Arn + InstanceProfile: + Value: !Ref WorkerProfile diff --git a/deploy/connect/aws/scanner/main.tf b/deploy/connect/aws/scanner/main.tf new file mode 100644 index 0000000..6e7de0e --- /dev/null +++ b/deploy/connect/aws/scanner/main.tf @@ -0,0 +1,82 @@ +variable "external_id" { + type = string + description = "Value the worker passes to sts:AssumeRole. Use the same value in every production stack." +} + +data "aws_iam_policy_document" "trust" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "worker" { + statement { + sid = "CopyAndAttach" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", + "ec2:DescribeVolumes", + "ec2:CopySnapshot", + "ec2:CreateVolume", + "ec2:CreateTags", + "ec2:AttachVolume", + "ec2:DetachVolume", + "ec2:DeleteVolume", + "ec2:DeleteSnapshot", + ] + resources = ["*"] + } + + statement { + sid = "AssumeProduction" + actions = ["sts:AssumeRole"] + resources = ["arn:aws:iam::*:role/opensourceom-snapshot"] + condition { + test = "StringEquals" + variable = "sts:ExternalId" + values = [var.external_id] + } + } + + statement { + sid = "Keys" + actions = [ + "kms:CreateGrant", + "kms:Decrypt", + "kms:Encrypt", + "kms:DescribeKey", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + ] + resources = ["*"] + } +} + +resource "aws_iam_role" "worker" { + name = "opensourceom-worker" + assume_role_policy = data.aws_iam_policy_document.trust.json +} + +resource "aws_iam_role_policy" "worker" { + name = "worker" + role = aws_iam_role.worker.id + policy = data.aws_iam_policy_document.worker.json +} + +resource "aws_iam_instance_profile" "worker" { + name = "opensourceom-worker" + role = aws_iam_role.worker.name +} + +output "role_arn" { + value = aws_iam_role.worker.arn +} + +output "instance_profile" { + value = aws_iam_instance_profile.worker.name +} diff --git a/deploy/connect/azure/production/apply.sh b/deploy/connect/azure/production/apply.sh new file mode 100644 index 0000000..37f8f56 --- /dev/null +++ b/deploy/connect/azure/production/apply.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Grant the scanner identity permission to snapshot this subscription. +# OpenTofu and Terraform use main.tf. +set -eu + +: "${AZURE_SUBSCRIPTION_ID:?set AZURE_SUBSCRIPTION_ID}" +: "${AZURE_LOCATION:?set AZURE_LOCATION}" +: "${SCANNER_PRINCIPAL_ID:?set SCANNER_PRINCIPAL_ID}" + +az group create --subscription "$AZURE_SUBSCRIPTION_ID" --name opensourceom-snapshots --location "$AZURE_LOCATION" + +role=$(cat <<'EOF' +{ + "Name": "OpenSourceOM snapshot", + "IsCustom": true, + "Description": "Create and delete OpenSourceOM workload snapshots. Does not attach a disk.", + "Actions": [ + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/images/read", + "Microsoft.Compute/galleries/images/versions/read", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.KeyVault/vaults/keys/wrap/action", + "Microsoft.KeyVault/vaults/keys/unwrap/action" + ], + "NotActions": [], + "AssignableScopes": ["/subscriptions/SUBSCRIPTION_ID"] +} +EOF +) +role=$(printf '%s' "$role" | sed "s#SUBSCRIPTION_ID#${AZURE_SUBSCRIPTION_ID}#") +printf '%s' "$role" > /tmp/opensourceom-snapshot-role.json +az role definition create --role-definition /tmp/opensourceom-snapshot-role.json >/dev/null 2>&1 \ + || az role definition update --role-definition /tmp/opensourceom-snapshot-role.json >/dev/null +az role assignment create \ + --subscription "$AZURE_SUBSCRIPTION_ID" \ + --assignee-object-id "$SCANNER_PRINCIPAL_ID" \ + --assignee-principal-type ServicePrincipal \ + --role "OpenSourceOM snapshot" +rm -f /tmp/opensourceom-snapshot-role.json diff --git a/deploy/connect/azure/production/main.tf b/deploy/connect/azure/production/main.tf new file mode 100644 index 0000000..2a77335 --- /dev/null +++ b/deploy/connect/azure/production/main.tf @@ -0,0 +1,46 @@ +variable "subscription_id" { + type = string +} + +variable "location" { + type = string +} + +variable "scanner_principal_id" { + type = string + description = "Object id of the scanner subscription's opensourceom-worker managed identity." +} + +resource "azurerm_resource_group" "snapshots" { + name = "opensourceom-snapshots" + location = var.location +} + +resource "azurerm_role_definition" "snapshot" { + name = "OpenSourceOM snapshot" + scope = "/subscriptions/${var.subscription_id}" + description = "Create and delete OpenSourceOM workload snapshots. Does not attach a disk." + + permissions { + actions = [ + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/images/read", + "Microsoft.Compute/galleries/images/versions/read", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.KeyVault/vaults/keys/wrap/action", + "Microsoft.KeyVault/vaults/keys/unwrap/action", + ] + } + + assignable_scopes = ["/subscriptions/${var.subscription_id}"] +} + +resource "azurerm_role_assignment" "snapshot" { + scope = "/subscriptions/${var.subscription_id}" + role_definition_id = azurerm_role_definition.snapshot.role_definition_resource_id + principal_id = var.scanner_principal_id +} diff --git a/deploy/connect/azure/scanner/apply.sh b/deploy/connect/azure/scanner/apply.sh new file mode 100644 index 0000000..44acc7e --- /dev/null +++ b/deploy/connect/azure/scanner/apply.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Create the scanner identity and let it attach disks in this resource group. +set -eu + +: "${AZURE_SUBSCRIPTION_ID:?set AZURE_SUBSCRIPTION_ID}" +: "${AZURE_LOCATION:?set AZURE_LOCATION}" +: "${AZURE_RESOURCE_GROUP:?set AZURE_RESOURCE_GROUP}" + +az identity create \ + --subscription "$AZURE_SUBSCRIPTION_ID" \ + --resource-group "$AZURE_RESOURCE_GROUP" \ + --name opensourceom-worker \ + --location "$AZURE_LOCATION" +principal=$(az identity show \ + --subscription "$AZURE_SUBSCRIPTION_ID" \ + --resource-group "$AZURE_RESOURCE_GROUP" \ + --name opensourceom-worker \ + --query principalId --output tsv) + +scope="/subscriptions/${AZURE_SUBSCRIPTION_ID}" +assignment="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AZURE_RESOURCE_GROUP}" +role=$(cat < /tmp/opensourceom-worker-role.json +az role definition create --role-definition /tmp/opensourceom-worker-role.json >/dev/null 2>&1 \ + || az role definition update --role-definition /tmp/opensourceom-worker-role.json >/dev/null +az role assignment create \ + --scope "$assignment" \ + --assignee-object-id "$principal" \ + --assignee-principal-type ServicePrincipal \ + --role "OpenSourceOM worker" +rm -f /tmp/opensourceom-worker-role.json +printf '%s\n' "$principal" diff --git a/deploy/connect/azure/scanner/main.tf b/deploy/connect/azure/scanner/main.tf new file mode 100644 index 0000000..783b294 --- /dev/null +++ b/deploy/connect/azure/scanner/main.tf @@ -0,0 +1,52 @@ +variable "subscription_id" { + type = string +} + +variable "location" { + type = string +} + +variable "resource_group" { + type = string + description = "Resource group of the worker VM. Disks are created here." +} + +resource "azurerm_user_assigned_identity" "worker" { + name = "opensourceom-worker" + location = var.location + resource_group_name = var.resource_group +} + +resource "azurerm_role_definition" "worker" { + name = "OpenSourceOM worker" + scope = "/subscriptions/${var.subscription_id}" + description = "Attach workload disks to the scanner VM." + + permissions { + actions = [ + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/images/read", + "Microsoft.Compute/galleries/images/versions/read", + "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.KeyVault/vaults/keys/wrap/action", + "Microsoft.KeyVault/vaults/keys/unwrap/action", + ] + } + + assignable_scopes = ["/subscriptions/${var.subscription_id}"] +} + +resource "azurerm_role_assignment" "worker" { + scope = "/subscriptions/${var.subscription_id}/resourceGroups/${var.resource_group}" + role_definition_id = azurerm_role_definition.worker.role_definition_resource_id + principal_id = azurerm_user_assigned_identity.worker.principal_id +} + +output "principal_id" { + value = azurerm_user_assigned_identity.worker.principal_id +} diff --git a/deploy/connect/gcp/production/apply.sh b/deploy/connect/gcp/production/apply.sh new file mode 100644 index 0000000..b28abbc --- /dev/null +++ b/deploy/connect/gcp/production/apply.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Grant the scanner service account permission to snapshot this project. +set -eu + +: "${GCP_PROJECT_ID:?set GCP_PROJECT_ID}" +: "${SCANNER_SERVICE_ACCOUNT:?set SCANNER_SERVICE_ACCOUNT}" + +permissions="compute.instances.list,compute.instances.get,compute.disks.list,compute.disks.createSnapshot,compute.regionDisks.createSnapshot,compute.snapshots.create,compute.snapshots.get,compute.snapshots.delete,compute.snapshots.useReadOnly,compute.images.list,compute.images.get,compute.images.useReadOnly,compute.zoneOperations.get,compute.globalOperations.get,cloudkms.cryptoKeyVersions.useToDecrypt,cloudkms.cryptoKeyVersions.useToEncrypt" + +if gcloud iam roles describe opensourceomSnapshot --project "$GCP_PROJECT_ID" >/dev/null 2>&1; then + gcloud iam roles update opensourceomSnapshot --project "$GCP_PROJECT_ID" --permissions "$permissions" +else + gcloud iam roles create opensourceomSnapshot --project "$GCP_PROJECT_ID" \ + --title "OpenSourceOM snapshot" \ + --description "Create and delete workload snapshots and grant read of boot images. Does not attach a disk." \ + --permissions "$permissions" +fi +gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \ + --member "serviceAccount:${SCANNER_SERVICE_ACCOUNT}" \ + --role "projects/${GCP_PROJECT_ID}/roles/opensourceomSnapshot" diff --git a/deploy/connect/gcp/production/main.tf b/deploy/connect/gcp/production/main.tf new file mode 100644 index 0000000..a6c8150 --- /dev/null +++ b/deploy/connect/gcp/production/main.tf @@ -0,0 +1,39 @@ +variable "project_id" { + type = string +} + +variable "scanner_service_account" { + type = string + description = "Email of the scanner project's opensourceom-worker service account." +} + +resource "google_project_iam_custom_role" "snapshot" { + role_id = "opensourceomSnapshot" + title = "OpenSourceOM snapshot" + description = "Create and delete workload snapshots and grant read of boot images. Does not attach a disk." + permissions = [ + "compute.instances.list", + "compute.instances.get", + "compute.disks.list", + "compute.disks.createSnapshot", + "compute.regionDisks.createSnapshot", + "compute.snapshots.create", + "compute.snapshots.get", + "compute.snapshots.delete", + "compute.snapshots.useReadOnly", + "compute.images.list", + "compute.images.get", + "compute.images.useReadOnly", + "compute.zoneOperations.get", + "compute.globalOperations.get", + "cloudkms.cryptoKeyVersions.useToDecrypt", + "cloudkms.cryptoKeyVersions.useToEncrypt", + ] + project = var.project_id +} + +resource "google_project_iam_member" "snapshot" { + project = var.project_id + role = google_project_iam_custom_role.snapshot.id + member = "serviceAccount:${var.scanner_service_account}" +} diff --git a/deploy/connect/gcp/scanner/apply.sh b/deploy/connect/gcp/scanner/apply.sh new file mode 100644 index 0000000..cd30bf5 --- /dev/null +++ b/deploy/connect/gcp/scanner/apply.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Create the scanner service account and let it attach disks in this project. +set -eu + +: "${GCP_PROJECT_ID:?set GCP_PROJECT_ID}" + +gcloud iam service-accounts create opensourceom-worker --project "$GCP_PROJECT_ID" \ + --display-name "OpenSourceOM worker" 2>/dev/null || true +email="opensourceom-worker@${GCP_PROJECT_ID}.iam.gserviceaccount.com" + +permissions="compute.instances.list,compute.instances.get,compute.disks.create,compute.disks.get,compute.disks.delete,compute.disks.useReadOnly,compute.instances.attachDisk,compute.instances.detachDisk,compute.zoneOperations.get,compute.globalOperations.get,cloudkms.cryptoKeyVersions.useToDecrypt,cloudkms.cryptoKeyVersions.useToEncrypt" + +if gcloud iam roles describe opensourceomWorker --project "$GCP_PROJECT_ID" >/dev/null 2>&1; then + gcloud iam roles update opensourceomWorker --project "$GCP_PROJECT_ID" --permissions "$permissions" +else + gcloud iam roles create opensourceomWorker --project "$GCP_PROJECT_ID" \ + --title "OpenSourceOM worker" \ + --description "Create a disk from a shared snapshot and attach it to the scanner instance." \ + --permissions "$permissions" +fi +gcloud projects add-iam-policy-binding "$GCP_PROJECT_ID" \ + --member "serviceAccount:${email}" \ + --role "projects/${GCP_PROJECT_ID}/roles/opensourceomWorker" +printf '%s\n' "$email" diff --git a/deploy/connect/gcp/scanner/main.tf b/deploy/connect/gcp/scanner/main.tf new file mode 100644 index 0000000..b587493 --- /dev/null +++ b/deploy/connect/gcp/scanner/main.tf @@ -0,0 +1,40 @@ +variable "project_id" { + type = string +} + +resource "google_service_account" "worker" { + account_id = "opensourceom-worker" + display_name = "OpenSourceOM worker" + project = var.project_id +} + +resource "google_project_iam_custom_role" "worker" { + role_id = "opensourceomWorker" + title = "OpenSourceOM worker" + description = "Create a disk from a shared snapshot and attach it to the scanner instance." + permissions = [ + "compute.instances.list", + "compute.instances.get", + "compute.disks.create", + "compute.disks.get", + "compute.disks.delete", + "compute.disks.useReadOnly", + "compute.instances.attachDisk", + "compute.instances.detachDisk", + "compute.zoneOperations.get", + "compute.globalOperations.get", + "cloudkms.cryptoKeyVersions.useToDecrypt", + "cloudkms.cryptoKeyVersions.useToEncrypt", + ] + project = var.project_id +} + +resource "google_project_iam_member" "worker" { + project = var.project_id + role = google_project_iam_custom_role.worker.id + member = "serviceAccount:${google_service_account.worker.email}" +} + +output "service_account" { + value = google_service_account.worker.email +} diff --git a/deploy/connect/permissions_test.go b/deploy/connect/permissions_test.go new file mode 100644 index 0000000..95dca93 --- /dev/null +++ b/deploy/connect/permissions_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package connect_test + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + awsv "github.com/OpenSourceOM/core/internal/workload/aws" + azurev "github.com/OpenSourceOM/core/internal/workload/azure" + gcpv "github.com/OpenSourceOM/core/internal/workload/gcp" +) + +func TestConnectTemplatesMatchPermissions(t *testing.T) { + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("caller") + } + root := filepath.Dir(file) + check(t, filepath.Join(root, "aws/production"), awsv.Permissions, []string{"ec2:AttachVolume", "ec2:CopySnapshot", "ec2:CreateVolume"}) + check(t, filepath.Join(root, "aws/scanner"), awsv.WorkerPermissions, []string{"ec2:CreateSnapshot", "ec2:ModifySnapshotAttribute"}) + check(t, filepath.Join(root, "azure/production"), azurev.Permissions, []string{"Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", "Microsoft.Compute/disks/write"}) + check(t, filepath.Join(root, "azure/scanner"), azurev.WorkerPermissions, []string{"Microsoft.Compute/snapshots/write"}) + check(t, filepath.Join(root, "gcp/production"), gcpv.Permissions, []string{"compute.instances.attachDisk", "compute.disks.create"}) + check(t, filepath.Join(root, "gcp/scanner"), gcpv.WorkerPermissions, []string{"compute.disks.createSnapshot", "compute.snapshots.create"}) +} + +func check(t *testing.T, dir string, want, forbid []string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var files []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + switch filepath.Ext(entry.Name()) { + case ".tf", ".yaml", ".sh", ".json": + files = append(files, filepath.Join(dir, entry.Name())) + } + } + if len(files) == 0 { + t.Fatalf("no templates in %s", dir) + } + for _, path := range files { + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + text := string(body) + for _, action := range want { + if !hasAction(text, action) { + t.Errorf("%s missing %s", path, action) + } + } + for _, action := range forbid { + if hasAction(text, action) { + t.Errorf("%s grants %s", path, action) + } + } + } +} + +func hasAction(body, action string) bool { + rest := body + for { + i := strings.Index(rest, action) + if i < 0 { + return false + } + end := i + len(action) + nextOK := end >= len(rest) || strings.ContainsRune("\"', \n\t]", rune(rest[end])) + prevOK := i == 0 || strings.ContainsRune("\"', [\n\t", rune(rest[i-1])) + if nextOK && prevOK { + return true + } + rest = rest[end:] + } +} diff --git a/deploy/helm/opensourceom/values.yaml b/deploy/helm/opensourceom/values.yaml index a6d0c86..429607d 100644 --- a/deploy/helm/opensourceom/values.yaml +++ b/deploy/helm/opensourceom/values.yaml @@ -59,10 +59,8 @@ resources: # Secret, or serviceAccountAuth). Kubernetes is off unless enabled; without # a kubeconfig Secret the chart creates a read-only ServiceAccount for this # cluster. `om scan demo` and the workload scans are not scheduled. -# The workload scan mounts disks and runs on a VM in the account: EC2 for -# `om scan workload aws` and `om scan workload image aws`, an Azure VM for -# `om scan workload azure` and `om scan workload image azure`, and a GCE -# instance for `om scan workload gcp` and `om scan workload image gcp`. +# The workload scan mounts disks on a VM in a scanner account, not in the +# account being scanned. The production grants are in deploy/connect. scan: schedule: "0 */6 * * *" successfulJobsHistoryLimit: 3 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6b1c6a1..76b46c1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -87,7 +87,7 @@ Further community rule packs (PCI and additional CIS mappings) stay open for con Phase 4 writes `packages` (CPE 2.3 names and package URLs) from persistent workload storage onto the workload node that enrichment already matches. The sources, in order, are AWS Linux root volumes, Azure VM and GCE root volumes, boot images, container image layers and function bundles, and shared filesystems (EFS, Azure Files, Filestore). The scan is opt-in. `om scan aws`, `om scan azure`, and `om scan gcp` stay read-only API collectors. -`om scan workload aws` is the AWS root-volume scan. `om scan workload azure` and `om scan workload gcp` are the same walk for Azure OS disks and GCE boot disks. `om scan workload image aws`, `om scan workload image azure`, and `om scan workload image gcp` read each boot image once and add those package identifiers to instances launched from it. A package that exists only on an instance disk stays on that instance. Each command runs as root on a worker in that account and region, mounts a copy read-only, and stores package identifiers only. A later API scan keeps `packages`. Windows, LVM, and non-root disks are skipped. Container layers, function bundles, and shared filesystems are still open. See [ADR 006](./adr/006-aws-root-volume-packages.md), [ADR 007](./adr/007-azure-gce-root-volume-packages.md), and [ADR 008](./adr/008-boot-image-packages.md). +`om scan workload aws` is the AWS root-volume scan. `om scan workload azure` and `om scan workload gcp` are the same walk for Azure OS disks and GCE boot disks. `om scan workload image aws`, `om scan workload image azure`, and `om scan workload image gcp` read each boot image once and add those package identifiers to instances launched from it. A package that exists only on an instance disk stays on that instance. Each command runs as root on a worker in a scanner account, subscription, or project. The production account grants snapshot and share. The worker mounts the copy and stores package identifiers only. A later API scan keeps `packages`. Windows, LVM, and non-root disks are skipped. Container layers, function bundles, and shared filesystems are still open. See [ADR 006](./adr/006-aws-root-volume-packages.md), [ADR 007](./adr/007-azure-gce-root-volume-packages.md), [ADR 008](./adr/008-boot-image-packages.md), and [ADR 009](./adr/009-scanner-account.md). The grants are in [deploy/connect](../deploy/connect/README.md). Phase 5 adds a finding for a secret pattern or a malware signature on storage that Phase 4 reader already opened. The finding stores the kind and the path. Object bodies and database rows are in neither phase. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 1974a5e..8f54353 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -64,7 +64,7 @@ Phase 4 is open. Cloud collectors record the VM, its exposure, and its identity. The phase ends when a package found in that storage is a CVE finding on the workload the graph already stores. A block volume is the first source. Boot images, container image layers, function bundles, and shared filesystems are the rest. -The scan is opt-in and separate from `om scan aws`, `om scan azure`, and `om scan gcp`. Those commands stay read-only. The workload scan runs a worker in the account and region, reads package metadata, and writes CPE 2.3 names and package URLs onto the existing workload. Snapshots and mounts are removed when the scan finishes, including when it fails. File contents stay on the storage. +The scan is opt-in and separate from `om scan aws`, `om scan azure`, and `om scan gcp`. Those commands stay read-only. The workload scan runs a worker in a scanner account, subscription, or project, reads package metadata, and writes CPE 2.3 names and package URLs onto the existing workload. Snapshots and mounts are removed when the scan finishes, including when it fails. File contents stay off the graph. The path, in order: @@ -74,9 +74,9 @@ The path, in order: - [ ] **Container image layers and function bundles** — layers for images already stored on Kubernetes workloads, plus Lambda deployment packages and the Azure and GCP equivalents. No volume attach — [#105](https://github.com/OpenSourceOM/core/issues/105) - [ ] **Shared filesystems** — EFS, Azure Files, and Filestore, when the application is installed there and a root-volume scan would miss it — [#106](https://github.com/OpenSourceOM/core/issues/106) -`om scan workload aws` reads Linux root volumes in the current AWS account. It runs as root on an EC2 instance in that account, snapshots the root volume, mounts the copy read-only, and writes CPE 2.3 names and package URLs onto `packages`. `om scan workload azure` does the same for managed OS disks from an Azure VM in that region, and `om scan workload gcp` does it for GCE boot disks from an instance in that region. A later `om scan aws`, `om scan azure`, or `om scan gcp` leaves `packages` in place. Windows, LVM, and disks other than the root volume are skipped. +`om scan workload aws` reads Linux root volumes in a production account from a worker in a scanner account. It assumes a snapshot role, shares the snapshot, copies it into the scanner account, and mounts that copy read-only. `om scan workload azure` and `om scan workload gcp` do the same from a scanner subscription or project. The grants are in `deploy/connect`. A later `om scan aws`, `om scan azure`, or `om scan gcp` leaves `packages` in place. Windows, LVM, and disks other than the root volume are skipped. See [ADR 009](./adr/009-scanner-account.md). -`om scan workload image aws` reads each AMI once, including an image no instance is running, and adds those package identifiers to every instance launched from it. `om scan workload image azure` and `om scan workload image gcp` do the same for Azure and GCE images. A package found only on an instance's own disk stays on that instance. Marketplace images the account cannot snapshot are skipped. The volume or disk created for the read is deleted. The image's own snapshot is not. +`om scan workload image aws` reads each AMI once, including an image no instance is running, and adds those package identifiers to every instance launched from it. `om scan workload image azure` and `om scan workload image gcp` do the same for Azure and GCE images. A package found only on an instance's own disk stays on that instance. Marketplace images the account cannot share are skipped. The volume or disk created for the read is deleted. The image's own snapshot is not. Object bodies and database rows stay out. They are customer data, not the installed software. A public bucket or database is already an exposure finding from the API scan. diff --git a/docs/adr/006-aws-root-volume-packages.md b/docs/adr/006-aws-root-volume-packages.md index 6b0135f..8ff9e64 100644 --- a/docs/adr/006-aws-root-volume-packages.md +++ b/docs/adr/006-aws-root-volume-packages.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # ADR 006: AWS Linux root-volume package inventory -**Status:** Accepted +**Status:** Accepted. The worker location is superseded by [ADR 009](./009-scanner-account.md). **Date:** 2026-10-14 **Phase:** 4 diff --git a/docs/adr/007-azure-gce-root-volume-packages.md b/docs/adr/007-azure-gce-root-volume-packages.md index 69ffa72..2635d27 100644 --- a/docs/adr/007-azure-gce-root-volume-packages.md +++ b/docs/adr/007-azure-gce-root-volume-packages.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # ADR 007: Azure and GCE root-volume package inventory -**Status:** Accepted +**Status:** Accepted. The worker location is superseded by [ADR 009](./009-scanner-account.md). **Date:** 2026-10-15 **Phase:** 4 diff --git a/docs/adr/008-boot-image-packages.md b/docs/adr/008-boot-image-packages.md index 6da271c..b421c72 100644 --- a/docs/adr/008-boot-image-packages.md +++ b/docs/adr/008-boot-image-packages.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # ADR 008: Boot-image package inventory -**Status:** Accepted +**Status:** Accepted. The worker location is superseded by [ADR 009](./009-scanner-account.md). **Date:** 2026-10-26 **Phase:** 4 diff --git a/docs/adr/009-scanner-account.md b/docs/adr/009-scanner-account.md new file mode 100644 index 0000000..bd28750 --- /dev/null +++ b/docs/adr/009-scanner-account.md @@ -0,0 +1,37 @@ + + +# ADR 009: Scanner account for workload disks + +**Status:** Accepted +**Date:** 2026-10-27 +**Phase:** 4 + +## Context + +[ADR 006](./006-aws-root-volume-packages.md), [ADR 007](./007-azure-gce-root-volume-packages.md), and [ADR 008](./008-boot-image-packages.md) mount a copy of a root volume or boot image on a worker in the account being scanned. That worker is compute inside the production account. A role can create a snapshot from anywhere. Attaching the disk still needs a machine, and that machine does not have to live in the account that owns the volume. + +## Decision + +The worker runs in a scanner account, subscription, or project. The account being scanned grants a role that can describe instances, create a snapshot, share it with the scanner, and delete snapshots this scan tagged. It does not grant attach, and it does not start an instance. + +`om scan workload aws` runs as root on an EC2 instance in the scanner account. It assumes the production role with an external id, creates the snapshot, and shares it. The worker copies the snapshot into the scanner account, encrypts the copy with a key in that account when one is set, attaches the copy, and reads package databases. It then deletes the copy and the source snapshot. An encrypted source requires `--kms-key`. The AMI scan shares the existing snapshot, copies it, and does not delete the AMI snapshot. + +`om scan workload azure` and `om scan workload gcp` use one credential that holds the production role and the scanner role. `AZURE_SUBSCRIPTION_ID` and `GCP_PROJECT_ID` are the production ids. Snapshots are created there. Disks are created and attached in the worker's subscription or project. Azure snapshots go in the `opensourceom-snapshots` resource group. The same split applies to boot images: the image stays in the production account, and the disk is created next to the worker. + +`deploy/connect` is the customer-facing grant. Terraform and OpenTofu share one module per side. AWS also has CloudFormation and a shell script. Azure has an `az` script. GCP has a `gcloud` script. A test checks those files against `Permissions` and `WorkerPermissions`. The API collectors stay read-only. The Helm chart still does not schedule the workload scan, because the worker is a VM in the scanner account. + +The graph stores package identifiers for the production account. File contents are not stored. Blocks are read in the scanner account and discarded after the package walk. + +## Consequences + +Each production account owner applies the production stack and adds the scanner principal to every customer-managed key that encrypts a root volume. The worker is tagged so a killed process can be cleaned up: `opensourceom:scan=workload` on snapshots and volumes, and `opensourceom-scan=workload` on GCP. A snapshot shared with the scanner account is unshared when the copy is deleted. Same-subscription and same-project runs still mount in that subscription or project; the AWS command refuses a snapshot role that lives in the scanner account. + +## References + +- [deploy/connect](../../deploy/connect/README.md) +- [ADR 006](./006-aws-root-volume-packages.md) +- [ADR 007](./007-azure-gce-root-volume-packages.md) +- [ADR 008](./008-boot-image-packages.md) diff --git a/go.mod b/go.mod index d719177..cf51368 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/anchore/go-rpmdb v0.2.2 github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/aws/aws-sdk-go-v2/config v1.33.12 + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.65.4 github.com/aws/aws-sdk-go-v2/service/ec2 v1.338.1 @@ -50,7 +51,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect diff --git a/internal/cmd/scan_workload.go b/internal/cmd/scan_workload.go index ff04343..a076bac 100644 --- a/internal/cmd/scan_workload.go +++ b/internal/cmd/scan_workload.go @@ -8,9 +8,9 @@ import "github.com/spf13/cobra" var scanWorkloadCmd = &cobra.Command{ Use: "workload", Short: "Inventory packages on workload storage", - Long: `Inventory packages on workload storage in the current account. + Long: `Inventory packages on workload storage. -Root volumes are "om scan workload aws", "om scan workload azure", and "om scan workload gcp". Boot images are "om scan workload image aws", "om scan workload image azure", and "om scan workload image gcp". Each runs as root on a worker in that account and region. om scan aws, om scan azure, and om scan gcp stay read-only.`, +Root volumes are "om scan workload aws", "om scan workload azure", and "om scan workload gcp". Boot images are "om scan workload image aws", "om scan workload image azure", and "om scan workload image gcp". Each runs as root on a worker in a scanner account, subscription, or project. It assumes a role in the account being scanned, shares a snapshot, and mounts the copy on the worker. om scan aws, om scan azure, and om scan gcp stay read-only. The grants are in deploy/connect.`, } func init() { diff --git a/internal/cmd/scan_workload_account.go b/internal/cmd/scan_workload_account.go new file mode 100644 index 0000000..6c6c604 --- /dev/null +++ b/internal/cmd/scan_workload_account.go @@ -0,0 +1,112 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + + azurev "github.com/OpenSourceOM/core/internal/workload/azure" + gcpv "github.com/OpenSourceOM/core/internal/workload/gcp" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +const defaultAzureSnapshotGroup = "opensourceom-snapshots" + +var ( + awsSnapshotRole string + awsExternalID string + awsKMSKey string + azureSnapshotGroup string +) + +func granted(title string, actions []string) string { + return title + ":\n " + strings.Join(actions, "\n ") +} + +func loadScannerAWS(ctx context.Context, region string) (localCfg, prodCfg aws.Config, scannerAccount, prodAccount string, err error) { + role := firstText(awsSnapshotRole, os.Getenv("OM_SNAPSHOT_ROLE_ARN")) + external := firstText(awsExternalID, os.Getenv("OM_EXTERNAL_ID")) + if role == "" || external == "" { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("set --snapshot-role and --external-id to the production role from deploy/connect/aws") + } + localCfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region)) + if err != nil { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("load aws config: %w", err) + } + localSTS := sts.NewFromConfig(localCfg) + ident, err := localSTS.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("scanner account: %w", err) + } + scannerAccount = aws.ToString(ident.Account) + provider := stscreds.NewAssumeRoleProvider(localSTS, role, func(o *stscreds.AssumeRoleOptions) { + o.ExternalID = aws.String(external) + o.RoleSessionName = "opensourceom-workload" + }) + prodCfg = localCfg.Copy() + prodCfg.Credentials = aws.NewCredentialsCache(provider) + prodIdent, err := sts.NewFromConfig(prodCfg).GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("assume %s: %w", role, err) + } + prodAccount = aws.ToString(prodIdent.Account) + if prodAccount == "" || scannerAccount == "" { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("aws account id is empty") + } + if prodAccount == scannerAccount { + return aws.Config{}, aws.Config{}, "", "", fmt.Errorf("snapshot role %s is in the scanner account; create it in the production account", role) + } + return localCfg, prodCfg, scannerAccount, prodAccount, nil +} + +func scannerKMSKey() string { + return firstText(awsKMSKey, os.Getenv("OM_SCANNER_KMS_KEY_ID")) +} + +func azureClients(prodSub, workerSub string) (prod, scanner *azurev.Client, group string, err error) { + prod, err = azurev.NewClient(prodSub) + if err != nil { + return nil, nil, "", err + } + if strings.EqualFold(workerSub, prodSub) { + return prod, nil, "", nil + } + scanner, err = azurev.NewClient(workerSub) + if err != nil { + return nil, nil, "", err + } + group = firstText(azureSnapshotGroup, defaultAzureSnapshotGroup) + return prod, scanner, group, nil +} + +func gcpClients(ctx context.Context, prodProject, workerProject string) (prod, scanner *gcpv.Client, err error) { + prod, err = gcpv.NewClient(ctx, prodProject) + if err != nil { + return nil, nil, err + } + if workerProject == prodProject { + return prod, nil, nil + } + scanner, err = gcpv.NewClient(ctx, workerProject) + if err != nil { + return nil, nil, err + } + return prod, scanner, nil +} + +func firstText(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} diff --git a/internal/cmd/scan_workload_aws.go b/internal/cmd/scan_workload_aws.go index ab49176..316692f 100644 --- a/internal/cmd/scan_workload_aws.go +++ b/internal/cmd/scan_workload_aws.go @@ -6,13 +6,11 @@ package cmd import ( "fmt" "os" - "strings" "time" awscol "github.com/OpenSourceOM/core/internal/collectors/aws" "github.com/OpenSourceOM/core/internal/workload" awsv "github.com/OpenSourceOM/core/internal/workload/aws" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/spf13/cobra" ) @@ -30,16 +28,15 @@ var scanWorkloadAWSCmd = &cobra.Command{ } func workloadAWSLong() string { - return `Inventory packages on Linux root volumes in the current AWS account. + return `Inventory packages on Linux root volumes in a production AWS account. -Run it as root on an EC2 instance in that account. The command snapshots each root volume, creates a volume from the snapshot in this instance's availability zone, attaches it, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan aws, then deletes the snapshot and the volume, including when a read fails. File contents stay on the volume. +Run it as root on an EC2 instance in the scanner account, in the same region. Pass --snapshot-role and --external-id for the role deploy/connect/aws creates in the production account. The command assumes that role, snapshots each root volume, and shares the snapshot with the scanner account. The worker copies it, creates a volume in this instance's availability zone, attaches it, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan aws, then deletes the copy and the source snapshot, including when a read fails. File contents stay off the graph. -om scan aws stays a read-only API scan. A later om scan aws keeps packages already stored on the workload. Snapshots and volumes are tagged opensourceom:scan=workload, which is how to find one left behind by a killed process. +An encrypted volume needs --kms-key, a key in the scanner account. The production key policy has to allow the scanner role to decrypt. om scan aws stays a read-only API scan. A later om scan aws keeps packages already stored on the workload. Snapshots and volumes are tagged opensourceom:scan=workload, which is how to find one left behind by a killed process. This command reads Linux root volumes. Windows instances are skipped. LVM and disks other than the root volume are skipped. Boot images are "om scan workload image aws". Container layers, function bundles, and shared filesystems are later scans. -IAM actions: - ` + strings.Join(awsv.Permissions, "\n ") +` + granted("Production role", awsv.Permissions) + "\n\n" + granted("Scanner role", awsv.WorkerPermissions) } func runScanWorkloadAWS(cmd *cobra.Command, args []string) error { @@ -62,17 +59,16 @@ func runScanWorkloadAWS(cmd *cobra.Command, args []string) error { } defer store.Close() - collector, err := awscol.NewCollector(ctx, cfg.AWSRegion) + localCfg, prodCfg, scannerAccount, prodAccount, err := loadScannerAWS(ctx, cfg.AWSRegion) if err != nil { return err } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(cfg.AWSRegion)) - if err != nil { - return fmt.Errorf("load aws config: %w", err) - } scanner := &awsv.Scanner{ - API: ec2.NewFromConfig(awsCfg), + API: ec2.NewFromConfig(localCfg), + Prod: ec2.NewFromConfig(prodCfg), + AccountID: scannerAccount, + KMSKeyID: scannerKMSKey(), Worker: awsv.Worker{ InstanceID: worker.InstanceID, AvailabilityZone: worker.AvailabilityZone, @@ -93,7 +89,7 @@ func runScanWorkloadAWS(cmd *cobra.Command, args []string) error { fmt.Printf("%s: skipped (%s)\n", label, res.Skipped) return nil } - nodeID := awscol.WorkloadNodeID(collector.AccountID, cfg.AWSRegion, res.InstanceID) + nodeID := awscol.WorkloadNodeID(prodAccount, cfg.AWSRegion, res.InstanceID) if err := store.SetWorkloadPackages(ctx, nodeID, res.Packages); err != nil { return err } @@ -115,5 +111,8 @@ func workloadLabel(instanceID, name string) string { func init() { scanWorkloadAWSCmd.Flags().DurationVar(&awsWorkloadTimeout, "timeout", 30*time.Minute, "time limit for each instance") scanWorkloadAWSCmd.Flags().StringArrayVar(&awsWorkloadInstances, "instance", nil, "EC2 instance to scan (repeat to scan several; the default is every instance in the region)") + scanWorkloadAWSCmd.Flags().StringVar(&awsSnapshotRole, "snapshot-role", "", "production role ARN from deploy/connect/aws (or OM_SNAPSHOT_ROLE_ARN)") + scanWorkloadAWSCmd.Flags().StringVar(&awsExternalID, "external-id", "", "external id the production role trusts (or OM_EXTERNAL_ID)") + scanWorkloadAWSCmd.Flags().StringVar(&awsKMSKey, "kms-key", "", "scanner account KMS key id or ARN for the copied snapshot (or OM_SCANNER_KMS_KEY_ID)") scanWorkloadCmd.AddCommand(scanWorkloadAWSCmd) } diff --git a/internal/cmd/scan_workload_azure.go b/internal/cmd/scan_workload_azure.go index 763934c..afb9b54 100644 --- a/internal/cmd/scan_workload_azure.go +++ b/internal/cmd/scan_workload_azure.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" "os" - "strings" "time" azurecol "github.com/OpenSourceOM/core/internal/collectors/azure" @@ -28,18 +27,17 @@ var scanWorkloadAzureCmd = &cobra.Command{ } func workloadAzureLong() string { - return `Inventory packages on Linux OS disks in the current Azure subscription. + return `Inventory packages on Linux OS disks in a production Azure subscription. -Run it as root on an Azure VM in that subscription. The command snapshots each managed OS disk in this VM's region, creates a disk from the snapshot in this VM's zone, attaches it, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan azure, then deletes the snapshot and the disk, including when a read fails. File contents stay on the disk. +Run it as root on an Azure VM in the scanner subscription, in the same region. AZURE_SUBSCRIPTION_ID is the production subscription. The grants are in deploy/connect/azure. The command snapshots each managed OS disk into the opensourceom-snapshots resource group, creates a disk from that snapshot in this VM's resource group and zone, attaches it, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan azure, then deletes the snapshot and the disk, including when a read fails. File contents stay off the graph. -om scan azure stays a read-only API scan. A later om scan azure keeps packages already stored on the workload. Snapshots and disks are tagged opensourceom:scan=workload, which is how to find one left behind by a killed process. The copies are created in this VM's resource group and cannot be exported. +om scan azure stays a read-only API scan. A later om scan azure keeps packages already stored on the workload. Snapshots and disks are tagged opensourceom:scan=workload, which is how to find one left behind by a killed process. The copies cannot be exported. This command reads Linux OS disks. Windows VMs are skipped. Ephemeral OS disks, unmanaged VHDs, LVM, and data disks are skipped. VMs in another region are skipped; run the command on a VM in that region. Boot images are "om scan workload image azure". The graph id matches om scan azure: azure:::workload:. -Azure RBAC actions: - ` + strings.Join(azurev.Permissions, "\n ") +` + granted("Production role", azurev.Permissions) + "\n\n" + granted("Scanner role", azurev.WorkerPermissions) } func runScanWorkloadAzure(cmd *cobra.Command, args []string) error { @@ -65,12 +63,19 @@ func runScanWorkloadAzure(cmd *cobra.Command, args []string) error { } defer store.Close() - client, err := azurev.NewClient(cfg.AzureSubscriptionID) + prod, scannerClient, group, err := azureClients(cfg.AzureSubscriptionID, worker.SubscriptionID) if err != nil { return err } + var disk azurev.API + if scannerClient != nil { + disk = scannerClient + } scanner := &azurev.Scanner{ - API: client, + API: prod, + Disk: disk, + ProdSubscription: cfg.AzureSubscriptionID, + SnapshotGroup: group, Worker: azurev.Worker{ Name: worker.Name, ResourceGroup: worker.ResourceGroup, @@ -115,5 +120,6 @@ func groupLabel(group, name string) string { func init() { scanWorkloadAzureCmd.Flags().DurationVar(&azureWorkloadTimeout, "timeout", 30*time.Minute, "time limit for each virtual machine") scanWorkloadAzureCmd.Flags().StringArrayVar(&azureWorkloadVMs, "instance", nil, "VM name, resource group/name, or resource id (repeat to scan several; the default is every VM in this region)") + scanWorkloadAzureCmd.Flags().StringVar(&azureSnapshotGroup, "snapshot-group", defaultAzureSnapshotGroup, "resource group in the production subscription that holds snapshots") scanWorkloadCmd.AddCommand(scanWorkloadAzureCmd) } diff --git a/internal/cmd/scan_workload_gcp.go b/internal/cmd/scan_workload_gcp.go index cf86967..0c53446 100644 --- a/internal/cmd/scan_workload_gcp.go +++ b/internal/cmd/scan_workload_gcp.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" "os" - "strings" "time" gcpcol "github.com/OpenSourceOM/core/internal/collectors/gcp" @@ -28,9 +27,9 @@ var scanWorkloadGCPCmd = &cobra.Command{ } func workloadGCPLong() string { - return `Inventory packages on Linux boot disks in the current GCP project. + return `Inventory packages on Linux boot disks in a production GCP project. -Run it as root on a GCE instance in that project. The command snapshots each persistent boot disk in this instance's region, creates a disk from the snapshot in this instance's zone, attaches it read-only, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan gcp, then deletes the snapshot and the disk, including when a read fails. File contents stay on the disk. +Run it as root on a GCE instance in the scanner project, in the same region. GCP_PROJECT_ID is the production project. The grants are in deploy/connect/gcp. The command snapshots each persistent boot disk in this instance's region, creates a disk from that snapshot in this instance's zone, attaches it read-only, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan gcp, then deletes the snapshot and the disk, including when a read fails. File contents stay off the graph. om scan gcp stays a read-only API scan. A later om scan gcp keeps packages already stored on the workload. Snapshots and disks are labeled opensourceom-scan=workload, which is how to find one left behind by a killed process. Label keys cannot contain a colon. @@ -38,8 +37,7 @@ This command reads Linux boot disks. Windows instances are skipped. Local SSD bo The graph id matches om scan gcp: gcp:::workload:. -IAM permissions: - ` + strings.Join(gcpv.Permissions, "\n ") +` + granted("Production role", gcpv.Permissions) + "\n\n" + granted("Scanner role", gcpv.WorkerPermissions) } func runScanWorkloadGCP(cmd *cobra.Command, args []string) error { @@ -65,12 +63,20 @@ func runScanWorkloadGCP(cmd *cobra.Command, args []string) error { } defer store.Close() - client, err := gcpv.NewClient(ctx, cfg.GCPProjectID) + prod, scannerClient, err := gcpClients(ctx, cfg.GCPProjectID, worker.ProjectID) if err != nil { return err } + var disk gcpv.API + project := "" + if scannerClient != nil { + disk = scannerClient + project = cfg.GCPProjectID + } scanner := &gcpv.Scanner{ - API: client, + API: prod, + Disk: disk, + ProdProject: project, Worker: gcpv.Worker{ Name: worker.Name, ProjectID: worker.ProjectID, diff --git a/internal/cmd/scan_workload_image.go b/internal/cmd/scan_workload_image.go index c88af8d..0b74baf 100644 --- a/internal/cmd/scan_workload_image.go +++ b/internal/cmd/scan_workload_image.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "os" - "strings" "time" awscol "github.com/OpenSourceOM/core/internal/collectors/aws" @@ -17,7 +16,6 @@ import ( awsv "github.com/OpenSourceOM/core/internal/workload/aws" azurev "github.com/OpenSourceOM/core/internal/workload/azure" gcpv "github.com/OpenSourceOM/core/internal/workload/gcp" - "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/spf13/cobra" ) @@ -61,46 +59,43 @@ var scanWorkloadImageGCPCmd = &cobra.Command{ } func workloadImageAWSLong() string { - return `Inventory packages on boot images in the current AWS account. + return `Inventory packages on boot images in a production AWS account. -Run it as root on an EC2 instance in that account. The command reads each AMI once: images this account owns, including ones no instance is running, and images that instances in this region were launched from. It creates a volume from the AMI root snapshot in this instance's availability zone, attaches it, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that AMI, then deletes the volume, including when a read fails. The AMI snapshot is left in place. File contents stay on the volume. +Run it as root on an EC2 instance in the scanner account, in the same region. Pass --snapshot-role and --external-id for the role deploy/connect/aws creates in the production account. The command reads each AMI once: images that account owns, including ones no instance is running, and images that instances in this region were launched from. It shares the AMI root snapshot with the scanner account, copies it, creates a volume in this instance's availability zone, attaches it, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that AMI, then deletes the copy and removes the share, including when a read fails. The AMI snapshot is left in place. File contents stay off the graph. -A package that exists only on an instance disk stays on that instance. This command does not copy it onto the image or onto another instance. om scan workload aws, which reads the instance disk, replaces packages with what is installed there. om scan aws stays a read-only API scan and keeps packages already stored. Volumes are tagged opensourceom:scan=workload. +An encrypted snapshot needs --kms-key, a key in the scanner account. A package that exists only on an instance disk stays on that instance. om scan workload aws replaces packages with what is installed on that instance disk. om scan aws stays a read-only API scan and keeps packages already stored. Volumes and snapshot copies are tagged opensourceom:scan=workload. -Windows images and instance-store images are skipped. Marketplace images this account cannot snapshot are skipped, and the command says so. Run om scan aws first so the instance nodes exist. +Windows images and instance-store images are skipped. Marketplace images this account cannot share are skipped, and the command says so. Run om scan aws first so the instance nodes exist. -IAM actions: - ` + strings.Join(awsv.ImagePermissions, "\n ") +` + granted("Production role", awsv.ImagePermissions) + "\n\n" + granted("Scanner role", awsv.WorkerPermissions) } func workloadImageAzureLong() string { - return `Inventory packages on boot images in the current Azure subscription. + return `Inventory packages on boot images in a production Azure subscription. -Run it as root on an Azure VM in that subscription. The command reads each managed image in this VM's region once, including images no VM is running, and each image a VM in this region was launched from. It creates a disk from the image in this VM's resource group and zone, attaches it, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that image, then deletes the disk, including when a read fails. The image is left in place. File contents stay on the disk. +Run it as root on an Azure VM in the scanner subscription, in the same region. AZURE_SUBSCRIPTION_ID is the production subscription. The grants are in deploy/connect/azure. The command reads each managed image in this region once, including images no VM is running, and each image a VM in this region was launched from. It creates a disk from the image in this VM's resource group and zone, attaches it, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that image, then deletes the disk, including when a read fails. The image is left in place. File contents stay off the graph. -A package that exists only on a VM disk stays on that VM. This command does not copy it onto the image or onto another VM. om scan workload azure replaces packages with what is installed on that OS disk. om scan azure stays a read-only API scan and keeps packages already stored. Disks are tagged opensourceom:scan=workload and cannot be exported. +A package that exists only on a VM disk stays on that VM. om scan workload azure replaces packages with what is installed on that OS disk. om scan azure stays a read-only API scan and keeps packages already stored. Disks are tagged opensourceom:scan=workload and cannot be exported. -Windows images are skipped. Marketplace images this subscription cannot snapshot are skipped, and the command says so. Run om scan azure first so the VM nodes exist. +Windows images are skipped. Marketplace images this subscription cannot read are skipped, and the command says so. Run om scan azure first so the VM nodes exist. The graph id matches om scan azure: azure:::workload:. -Azure RBAC actions: - ` + strings.Join(azurev.ImagePermissions, "\n ") +` + granted("Production role", azurev.ImagePermissions) + "\n\n" + granted("Scanner role", azurev.WorkerPermissions) } func workloadImageGCPLong() string { - return `Inventory packages on boot images in the current GCP project. + return `Inventory packages on boot images in a production GCP project. -Run it as root on a GCE instance in that project. The command reads each project image once, including images no instance in this region is running, and each image an instance in this region was launched from. It creates a disk from the image in this instance's zone, attaches it read-only, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that image, then deletes the disk, including when a read fails. The image is left in place. File contents stay on the disk. +Run it as root on a GCE instance in the scanner project, in the same region. GCP_PROJECT_ID is the production project. The grants are in deploy/connect/gcp. The command reads each project image once, including images no instance in this region is running, and each image an instance in this region was launched from. It creates a disk from the image in this instance's zone, attaches it read-only, and mounts it read-only. It adds those CPE 2.3 names and package URLs to packages on every workload launched from that image, then deletes the disk, including when a read fails. The image is left in place. File contents stay off the graph. -A package that exists only on an instance disk stays on that instance. This command does not copy it onto the image or onto another instance. om scan workload gcp replaces packages with what is installed on that boot disk. om scan gcp stays a read-only API scan and keeps packages already stored. Disks are labeled opensourceom-scan=workload. Label keys cannot contain a colon. +A package that exists only on an instance disk stays on that instance. om scan workload gcp replaces packages with what is installed on that boot disk. om scan gcp stays a read-only API scan and keeps packages already stored. Disks are labeled opensourceom-scan=workload. Label keys cannot contain a colon. -Windows images are skipped. Marketplace images this project cannot snapshot are skipped, and the command says so. Run om scan gcp first so the instance nodes exist. +Windows images are skipped. Marketplace images this project cannot read are skipped, and the command says so. Run om scan gcp first so the instance nodes exist. The graph id matches om scan gcp: gcp:::workload:. -IAM permissions: - ` + strings.Join(gcpv.ImagePermissions, "\n ") +` + granted("Production role", gcpv.ImagePermissions) + "\n\n" + granted("Scanner role", gcpv.WorkerPermissions) } func runScanWorkloadImageAWS(cmd *cobra.Command, args []string) error { @@ -123,17 +118,16 @@ func runScanWorkloadImageAWS(cmd *cobra.Command, args []string) error { } defer store.Close() - collector, err := awscol.NewCollector(ctx, cfg.AWSRegion) + localCfg, prodCfg, scannerAccount, prodAccount, err := loadScannerAWS(ctx, cfg.AWSRegion) if err != nil { return err } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(cfg.AWSRegion)) - if err != nil { - return fmt.Errorf("load aws config: %w", err) - } scanner := &awsv.Scanner{ - API: ec2.NewFromConfig(awsCfg), + API: ec2.NewFromConfig(localCfg), + Prod: ec2.NewFromConfig(prodCfg), + AccountID: scannerAccount, + KMSKeyID: scannerKMSKey(), Worker: awsv.Worker{ InstanceID: worker.InstanceID, AvailabilityZone: worker.AvailabilityZone, @@ -159,7 +153,7 @@ func runScanWorkloadImageAWS(cmd *cobra.Command, args []string) error { } var failed []error for _, inst := range res.Instances { - nodeID := awscol.WorkloadNodeID(collector.AccountID, cfg.AWSRegion, inst.ID) + nodeID := awscol.WorkloadNodeID(prodAccount, cfg.AWSRegion, inst.ID) if err := store.MergeWorkloadPackages(ctx, nodeID, res.Packages); err != nil { failed = append(failed, err) continue @@ -196,12 +190,17 @@ func runScanWorkloadImageAzure(cmd *cobra.Command, args []string) error { } defer store.Close() - client, err := azurev.NewClient(cfg.AzureSubscriptionID) + prod, scannerClient, _, err := azureClients(cfg.AzureSubscriptionID, worker.SubscriptionID) if err != nil { return err } + var disk azurev.ImageAPI + if scannerClient != nil { + disk = scannerClient + } scanner := &azurev.ImageScanner{ - API: client, + API: prod, + Disk: disk, Worker: azurev.Worker{ Name: worker.Name, ResourceGroup: worker.ResourceGroup, @@ -270,12 +269,17 @@ func runScanWorkloadImageGCP(cmd *cobra.Command, args []string) error { } defer store.Close() - client, err := gcpv.NewClient(ctx, cfg.GCPProjectID) + prod, scannerClient, err := gcpClients(ctx, cfg.GCPProjectID, worker.ProjectID) if err != nil { return err } + var disk gcpv.ImageAPI + if scannerClient != nil { + disk = scannerClient + } scanner := &gcpv.ImageScanner{ - API: client, + API: prod, + Disk: disk, Worker: gcpv.Worker{ Name: worker.Name, ProjectID: worker.ProjectID, @@ -322,8 +326,12 @@ func runScanWorkloadImageGCP(cmd *cobra.Command, args []string) error { func init() { scanWorkloadImageAWSCmd.Flags().DurationVar(&awsImageTimeout, "timeout", 30*time.Minute, "time limit for each image") scanWorkloadImageAWSCmd.Flags().StringArrayVar(&awsImageIDs, "image", nil, "AMI id or name (repeat to scan several; the default is every owned image and every image an instance in this region uses)") + scanWorkloadImageAWSCmd.Flags().StringVar(&awsSnapshotRole, "snapshot-role", "", "production role ARN from deploy/connect/aws (or OM_SNAPSHOT_ROLE_ARN)") + scanWorkloadImageAWSCmd.Flags().StringVar(&awsExternalID, "external-id", "", "external id the production role trusts (or OM_EXTERNAL_ID)") + scanWorkloadImageAWSCmd.Flags().StringVar(&awsKMSKey, "kms-key", "", "scanner account KMS key id or ARN for the copied snapshot (or OM_SCANNER_KMS_KEY_ID)") scanWorkloadImageAzureCmd.Flags().DurationVar(&azureImageTimeout, "timeout", 30*time.Minute, "time limit for each image") scanWorkloadImageAzureCmd.Flags().StringArrayVar(&azureImageIDs, "image", nil, "image name or resource id (repeat to scan several; the default is every image in this region)") + scanWorkloadImageAzureCmd.Flags().StringVar(&azureSnapshotGroup, "snapshot-group", defaultAzureSnapshotGroup, "resource group in the production subscription that holds snapshots") scanWorkloadImageGCPCmd.Flags().DurationVar(&gcpImageTimeout, "timeout", 30*time.Minute, "time limit for each image") scanWorkloadImageGCPCmd.Flags().StringArrayVar(&gcpImageIDs, "image", nil, "image name or resource link (repeat to scan several; the default is every project image and every image an instance in this region uses)") scanWorkloadImageCmd.AddCommand(scanWorkloadImageAWSCmd, scanWorkloadImageAzureCmd, scanWorkloadImageGCPCmd) diff --git a/internal/cmd/scan_workload_test.go b/internal/cmd/scan_workload_test.go index b5921cd..5dd56bf 100644 --- a/internal/cmd/scan_workload_test.go +++ b/internal/cmd/scan_workload_test.go @@ -28,19 +28,19 @@ func TestScanWorkloadRegistered(t *testing.T) { t.Fatal("om scan workload runs a scan; name the cloud") } aws := commandByName(t, workloadCmd, "aws") - for _, action := range awsv.Permissions { + for _, action := range append(append([]string{}, awsv.Permissions...), awsv.WorkerPermissions...) { if !strings.Contains(aws.Long, action) { t.Errorf("aws help missing %s", action) } } azure := commandByName(t, workloadCmd, "azure") - for _, action := range azurev.Permissions { + for _, action := range append(append([]string{}, azurev.Permissions...), azurev.WorkerPermissions...) { if !strings.Contains(azure.Long, action) { t.Errorf("azure help missing %s", action) } } gcp := commandByName(t, workloadCmd, "gcp") - for _, action := range gcpv.Permissions { + for _, action := range append(append([]string{}, gcpv.Permissions...), gcpv.WorkerPermissions...) { if !strings.Contains(gcp.Long, action) { t.Errorf("gcp help missing %s", action) } @@ -50,19 +50,19 @@ func TestScanWorkloadRegistered(t *testing.T) { t.Fatal("om scan workload image runs a scan; name the cloud") } awsImage := commandByName(t, image, "aws") - for _, action := range awsv.ImagePermissions { + for _, action := range append(append([]string{}, awsv.ImagePermissions...), awsv.WorkerPermissions...) { if !strings.Contains(awsImage.Long, action) { t.Errorf("image aws help missing %s", action) } } azureImage := commandByName(t, image, "azure") - for _, action := range azurev.ImagePermissions { + for _, action := range append(append([]string{}, azurev.ImagePermissions...), azurev.WorkerPermissions...) { if !strings.Contains(azureImage.Long, action) { t.Errorf("image azure help missing %s", action) } } gcpImage := commandByName(t, image, "gcp") - for _, action := range gcpv.ImagePermissions { + for _, action := range append(append([]string{}, gcpv.ImagePermissions...), gcpv.WorkerPermissions...) { if !strings.Contains(gcpImage.Long, action) { t.Errorf("image gcp help missing %s", action) } diff --git a/internal/workload/aws/account.go b/internal/workload/aws/account.go new file mode 100644 index 0000000..07340a7 --- /dev/null +++ b/internal/workload/aws/account.go @@ -0,0 +1,155 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +func (s *Scanner) crossAccount() bool { return s.AccountID != "" } + +func (s *Scanner) prodAPI() ec2API { + if s.Prod != nil { + return s.Prod + } + return s.API +} + +func (s *Scanner) instanceAPI() ec2API { + if s.crossAccount() { + return s.prodAPI() + } + return s.API +} + +func (s *Scanner) localWorker(ctx context.Context) (ec2types.Instance, error) { + input := &ec2.DescribeInstancesInput{InstanceIds: []string{s.Worker.InstanceID}} + for { + page, err := s.API.DescribeInstances(ctx, input) + if err != nil { + return ec2types.Instance{}, fmt.Errorf("describe worker: %w", err) + } + for _, reservation := range page.Reservations { + for _, inst := range reservation.Instances { + if aws.ToString(inst.InstanceId) == s.Worker.InstanceID { + return inst, nil + } + } + } + if page.NextToken == nil || *page.NextToken == "" { + return ec2types.Instance{}, fmt.Errorf("worker %s is not in the scanner account", s.Worker.InstanceID) + } + input.NextToken = page.NextToken + } +} + +// stageSnapshot shares a production snapshot with the scanner account and +// copies it there. The returned id is what CreateVolume attaches. cleanup +// deletes the copy and removes the share. The caller deletes the source +// snapshot when the scan created it. +func (s *Scanner) stageSnapshot(ctx context.Context, snapID, owner string) (string, func(), error) { + if !s.crossAccount() { + return snapID, nil, nil + } + if s.Worker.Region == "" { + return "", nil, fmt.Errorf("worker region is required to copy the snapshot") + } + encrypted, err := s.snapshotEncrypted(ctx, snapID) + if err != nil { + return "", nil, err + } + if encrypted && s.KMSKeyID == "" { + return "", nil, fmt.Errorf("snapshot %s is encrypted; set --kms-key to a key in the scanner account", snapID) + } + if err := s.shareSnapshot(ctx, snapID, true); err != nil { + return "", nil, err + } + copyID, err := s.copySnapshot(ctx, snapID, encrypted, owner) + if err != nil { + s.note(s.shareSnapshot(context.Background(), snapID, false)) + return "", nil, err + } + if err := s.waitLocalSnapshot(ctx, copyID); err != nil { + s.deleteLocalSnapshot(copyID) + s.note(s.shareSnapshot(context.Background(), snapID, false)) + return "", nil, err + } + return copyID, func() { + s.deleteLocalSnapshot(copyID) + s.note(s.shareSnapshot(context.Background(), snapID, false)) + }, nil +} + +func (s *Scanner) snapshotEncrypted(ctx context.Context, id string) (bool, error) { + out, err := s.prodAPI().DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{SnapshotIds: []string{id}}) + if err != nil { + return false, fmt.Errorf("describe snapshot: %w", err) + } + if len(out.Snapshots) == 0 { + return false, fmt.Errorf("snapshot %s disappeared", id) + } + return aws.ToBool(out.Snapshots[0].Encrypted), nil +} + +func (s *Scanner) shareSnapshot(ctx context.Context, id string, add bool) error { + perm := &ec2types.CreateVolumePermissionModifications{} + user := []ec2types.CreateVolumePermission{{UserId: aws.String(s.AccountID)}} + if add { + perm.Add = user + } else { + perm.Remove = user + } + callCtx := ctx + if !add { + var cancel context.CancelFunc + callCtx, cancel = context.WithTimeout(context.Background(), time.Minute) + defer cancel() + } + _, err := s.prodAPI().ModifySnapshotAttribute(callCtx, &ec2.ModifySnapshotAttributeInput{ + SnapshotId: aws.String(id), + Attribute: ec2types.SnapshotAttributeNameCreateVolumePermission, + CreateVolumePermission: perm, + }) + if err != nil && !add && isNotFound(err) { + return nil + } + if err != nil { + if add { + return fmt.Errorf("share snapshot: %w", err) + } + return fmt.Errorf("unshare snapshot %s: %w", id, err) + } + return nil +} + +func (s *Scanner) copySnapshot(ctx context.Context, sourceID string, encrypted bool, owner string) (string, error) { + in := &ec2.CopySnapshotInput{ + Description: aws.String("OpenSourceOM workload package inventory"), + SourceRegion: aws.String(s.Worker.Region), + SourceSnapshotId: aws.String(sourceID), + TagSpecifications: []ec2types.TagSpecification{{ + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: scanTags(owner), + }}, + } + if encrypted || s.KMSKeyID != "" { + in.Encrypted = aws.Bool(true) + in.KmsKeyId = aws.String(s.KMSKeyID) + } + out, err := s.API.CopySnapshot(ctx, in) + if err != nil { + return "", fmt.Errorf("copy snapshot: %w", err) + } + id := aws.ToString(out.SnapshotId) + if id == "" { + return "", fmt.Errorf("copy snapshot returned an empty snapshot id") + } + return id, nil +} diff --git a/internal/workload/aws/identity.go b/internal/workload/aws/identity.go index 51a6f38..65d0754 100644 --- a/internal/workload/aws/identity.go +++ b/internal/workload/aws/identity.go @@ -22,7 +22,7 @@ func LookupWorker(ctx context.Context) (Worker, error) { client := imds.NewFromConfig(cfg) id, err := metadata(ctx, client, "instance-id") if err != nil { - return Worker{}, fmt.Errorf("om scan workload aws runs on an EC2 instance in this account: %w", err) + return Worker{}, fmt.Errorf("om scan workload aws runs on an EC2 instance in the scanner account: %w", err) } az, err := metadata(ctx, client, "placement/availability-zone") if err != nil { diff --git a/internal/workload/aws/image.go b/internal/workload/aws/image.go index 3f29df8..4f39194 100644 --- a/internal/workload/aws/image.go +++ b/internal/workload/aws/image.go @@ -18,22 +18,17 @@ import ( "github.com/aws/smithy-go" ) -// ImagePermissions are the IAM actions om scan workload image adds. -// The AMI's own snapshot is not deleted. kms:CreateGrant and kms:Decrypt are -// the grant an encrypted image snapshot needs when the volume is created. +// ImagePermissions are the IAM actions the production role needs for an AMI. +// The AMI's own snapshot is shared, copied into the scanner account, and left +// in place. CreateVolume and DeleteSnapshot belong to WorkerPermissions. var ImagePermissions = []string{ "ec2:DescribeInstances", "ec2:DescribeImages", - "ec2:CreateVolume", - "ec2:DescribeVolumes", - "ec2:AttachVolume", - "ec2:DetachVolume", - "ec2:DeleteVolume", - "ec2:CreateTags", + "ec2:DescribeSnapshots", + "ec2:ModifySnapshotAttribute", "kms:CreateGrant", "kms:Decrypt", "kms:DescribeKey", - "kms:GenerateDataKeyWithoutPlaintext", "kms:ReEncryptFrom", "kms:ReEncryptTo", } @@ -76,16 +71,23 @@ func (s *Scanner) ScanImages(ctx context.Context, only []string, write func(Imag return err } var worker ec2types.Instance - found := false - for _, inst := range instances { - if aws.ToString(inst.InstanceId) == s.Worker.InstanceID { - worker = inst - found = true - break + if s.crossAccount() { + worker, err = s.localWorker(ctx) + if err != nil { + return err + } + } else { + found := false + for _, inst := range instances { + if aws.ToString(inst.InstanceId) == s.Worker.InstanceID { + worker = inst + found = true + break + } + } + if !found { + return fmt.Errorf("worker %s is not in this region", s.Worker.InstanceID) } - } - if !found { - return fmt.Errorf("worker %s is not in this region", s.Worker.InstanceID) } images, err := s.bootImages(ctx, instances, only) if err != nil { @@ -200,8 +202,12 @@ func selectBootImages(images []bootImage, only []string) ([]bootImage, error) { func (s *Scanner) describeImages(ctx context.Context, input *ec2.DescribeImagesInput) ([]ec2types.Image, error) { var out []ec2types.Image + api := s.API + if s.crossAccount() { + api = s.prodAPI() + } for { - page, err := s.API.DescribeImages(ctx, input) + page, err := api.DescribeImages(ctx, input) if err != nil { return nil, err } @@ -238,7 +244,19 @@ func (s *Scanner) scanImage(ctx context.Context, item bootImage, instances []ec2 s.OnStart(id, res.Name) } - volID, err := s.createImageVolume(ctx, snapID, id) + readID, cleanup, err := s.stageSnapshot(ctx, snapID, id) + if cleanup != nil { + defer cleanup() + } + if err != nil { + if cannotSnapshotImage(err) { + res.Skipped = workload.ErrImageDenied.Error() + return res, nil + } + return ImageResult{}, fmt.Errorf("%s: %w", id, err) + } + + volID, err := s.createImageVolume(ctx, readID, id) if volID != "" { defer s.deleteVolume(volID) } diff --git a/internal/workload/aws/image_test.go b/internal/workload/aws/image_test.go index 7ac42c7..49b3fb5 100644 --- a/internal/workload/aws/image_test.go +++ b/internal/workload/aws/image_test.go @@ -16,14 +16,17 @@ import ( ) func TestImagePermissionsDoNotDeleteTheImageSnapshot(t *testing.T) { - if !slices.Contains(ImagePermissions, "ec2:DescribeImages") || !slices.Contains(ImagePermissions, "ec2:CreateVolume") { + if !slices.Contains(ImagePermissions, "ec2:DescribeImages") || !slices.Contains(ImagePermissions, "ec2:ModifySnapshotAttribute") { t.Fatalf("permissions = %v", ImagePermissions) } - for _, action := range []string{"ec2:CreateSnapshot", "ec2:DeleteSnapshot"} { + for _, action := range []string{"ec2:CreateSnapshot", "ec2:DeleteSnapshot", "ec2:CreateVolume", "ec2:AttachVolume"} { if slices.Contains(ImagePermissions, action) { t.Errorf("image scan must not %s", action) } } + if !slices.Contains(WorkerPermissions, "ec2:CreateVolume") || !slices.Contains(WorkerPermissions, "ec2:CopySnapshot") { + t.Fatalf("worker = %v", WorkerPermissions) + } } func TestScanImageOnceAppliesTheSamePackages(t *testing.T) { @@ -217,6 +220,49 @@ func TestScanImageDeletesVolumeWhenWalkFails(t *testing.T) { } } +func TestScanImageCopiesInTheScannerAccount(t *testing.T) { + web := linuxInstance("i-web", "web", "vol-web") + web.ImageId = aws.String("ami-golden") + prod := newFake([]ec2types.Instance{web}) + prod.images = []ec2types.Image{testAMI("ami-golden", "golden", "snap-golden")} + prod.snaps["snap-golden"] = &snapState{state: ec2types.SnapshotStateCompleted} + worker := linuxInstance("i-worker", "worker", "vol-worker") + worker.BlockDeviceMappings = append(worker.BlockDeviceMappings, ec2types.InstanceBlockDeviceMapping{ + DeviceName: aws.String("/dev/sdf"), + Ebs: &ec2types.EbsInstanceBlockDevice{VolumeId: aws.String("vol-extra")}, + }) + local := newFake([]ec2types.Instance{worker}) + scanner := testScanner(t, local, func(context.Context, string) (string, func() error, error) { + dir := t.TempDir() + writeRoot(t, dir) + return dir, func() error { return nil }, nil + }, workload.Walk) + scanner.Prod = prod + scanner.AccountID = "999988887777" + scanner.Worker.Region = "us-east-1" + + var got []ImageResult + if err := scanner.ScanImages(context.Background(), nil, func(res ImageResult) error { + got = append(got, res) + return nil + }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ImageID != "ami-golden" || got[0].Skipped != "" { + t.Fatalf("results = %#v", got) + } + if slices.Contains(prod.log, "DeleteSnapshot") || slices.Contains(prod.log, "CreateVolume") || slices.Contains(prod.log, "AttachVolume") { + t.Fatalf("production account changed the image snapshot: %v", prod.log) + } + if !slices.Contains(prod.log, "ModifySnapshotAttribute") { + t.Fatalf("prod log = %v", prod.log) + } + assertOrder(t, local.log, "CopySnapshot", "CreateVolume", "AttachVolume", "DeleteVolume", "DeleteSnapshot") + if _, ok := prod.snaps["snap-golden"]; !ok { + t.Fatal("AMI snapshot was deleted") + } +} + func imageInstances() []ec2types.Instance { worker := linuxInstance("i-worker", "worker", "vol-worker") worker.ImageId = aws.String("ami-worker") diff --git a/internal/workload/aws/scan.go b/internal/workload/aws/scan.go index 9935556..ce7cea4 100644 --- a/internal/workload/aws/scan.go +++ b/internal/workload/aws/scan.go @@ -16,21 +16,44 @@ import ( "github.com/aws/smithy-go" ) -// Permissions are the IAM actions om scan workload aws adds. -// kms:CreateGrant and kms:Decrypt are the grant an encrypted root volume needs. +// Permissions are the IAM actions of the role in the account being scanned. +// That role creates and shares snapshots. It does not attach a volume. +// kms:CreateGrant and kms:Decrypt are the grant an encrypted root volume needs +// when the snapshot is created. The key policy has to name the scanner role too. var Permissions = []string{ "ec2:DescribeInstances", - "ec2:CreateSnapshot", + "ec2:DescribeImages", "ec2:DescribeSnapshots", + "ec2:DescribeVolumes", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:ModifySnapshotAttribute", "ec2:DeleteSnapshot", - "ec2:CreateVolume", + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", +} + +// WorkerPermissions are the IAM actions of the instance profile in the scanner +// account. The worker copies the shared snapshot, attaches the volume, and +// deletes the copy. kms:Encrypt is the scanner account key used for that copy. +var WorkerPermissions = []string{ + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", "ec2:DescribeVolumes", + "ec2:CopySnapshot", + "ec2:CreateVolume", + "ec2:CreateTags", "ec2:AttachVolume", "ec2:DetachVolume", "ec2:DeleteVolume", - "ec2:CreateTags", + "ec2:DeleteSnapshot", + "sts:AssumeRole", "kms:CreateGrant", "kms:Decrypt", + "kms:Encrypt", "kms:DescribeKey", "kms:GenerateDataKeyWithoutPlaintext", "kms:ReEncryptFrom", @@ -53,15 +76,22 @@ type Result struct { } // Scanner snapshots Linux root volumes, reads them, and deletes what it created. +// API is the scanner account, where the worker attaches the volume. +// Prod is the account being scanned. AccountID is the scanner account id the +// snapshot is shared with. When AccountID is empty, Prod is unused and the +// volume is created from the snapshot in the same account as API. type Scanner struct { - API ec2API - Worker Worker - Only []string - Timeout time.Duration - Poll time.Duration - Mount func(ctx context.Context, volumeID string) (root string, cleanup func() error, err error) - Walk func(root string) ([]string, error) - OnStart func(instanceID, name string) + API ec2API + Prod ec2API + AccountID string + KMSKeyID string + Worker Worker + Only []string + Timeout time.Duration + Poll time.Duration + Mount func(ctx context.Context, volumeID string) (root string, cleanup func() error, err error) + Walk func(root string) ([]string, error) + OnStart func(instanceID, name string) cleanupErrs []error } @@ -77,6 +107,8 @@ type ec2API interface { DetachVolume(context.Context, *ec2.DetachVolumeInput, ...func(*ec2.Options)) (*ec2.DetachVolumeOutput, error) DeleteVolume(context.Context, *ec2.DeleteVolumeInput, ...func(*ec2.Options)) (*ec2.DeleteVolumeOutput, error) DeleteSnapshot(context.Context, *ec2.DeleteSnapshotInput, ...func(*ec2.Options)) (*ec2.DeleteSnapshotOutput, error) + ModifySnapshotAttribute(context.Context, *ec2.ModifySnapshotAttributeInput, ...func(*ec2.Options)) (*ec2.ModifySnapshotAttributeOutput, error) + CopySnapshot(context.Context, *ec2.CopySnapshotInput, ...func(*ec2.Options)) (*ec2.CopySnapshotOutput, error) } var _ ec2API = (*ec2.Client)(nil) @@ -125,9 +157,18 @@ func (s *Scanner) list(ctx context.Context) ([]ec2types.Instance, ec2types.Insta for _, inst := range instances { byID[aws.ToString(inst.InstanceId)] = inst } - worker, ok := byID[s.Worker.InstanceID] - if !ok { - return nil, ec2types.Instance{}, fmt.Errorf("worker %s is not in this region", s.Worker.InstanceID) + var worker ec2types.Instance + if s.crossAccount() { + worker, err = s.localWorker(ctx) + if err != nil { + return nil, ec2types.Instance{}, err + } + } else { + found, ok := byID[s.Worker.InstanceID] + if !ok { + return nil, ec2types.Instance{}, fmt.Errorf("worker %s is not in this region", s.Worker.InstanceID) + } + worker = found } if len(s.Only) == 0 { return instances, worker, nil @@ -150,7 +191,7 @@ func (s *Scanner) describe(ctx context.Context, ids []string) ([]ec2types.Instan } var out []ec2types.Instance for { - page, err := s.API.DescribeInstances(ctx, input) + page, err := s.instanceAPI().DescribeInstances(ctx, input) if err != nil { return nil, fmt.Errorf("describe instances: %w", err) } @@ -195,8 +236,15 @@ func (s *Scanner) scanInstance(ctx context.Context, inst, worker ec2types.Instan if err := s.waitSnapshot(ctx, snapID); err != nil { return Result{}, fmt.Errorf("%s: %w", id, err) } + readID, cleanup, err := s.stageSnapshot(ctx, snapID, id) + if cleanup != nil { + defer cleanup() + } + if err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } - volID, err := s.createVolume(ctx, snapID, id) + volID, err := s.createVolume(ctx, readID, id) if err != nil { return Result{}, fmt.Errorf("%s: %w", id, err) } @@ -245,7 +293,7 @@ func (s *Scanner) scanInstance(ctx context.Context, inst, worker ec2types.Instan } func (s *Scanner) createSnapshot(ctx context.Context, volumeID, instanceID string) (string, error) { - out, err := s.API.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{ + out, err := s.prodAPI().CreateSnapshot(ctx, &ec2.CreateSnapshotInput{ VolumeId: aws.String(volumeID), Description: aws.String("OpenSourceOM workload package inventory"), TagSpecifications: []ec2types.TagSpecification{{ @@ -292,8 +340,16 @@ func scanTags(instanceID string) []ec2types.Tag { } func (s *Scanner) waitSnapshot(ctx context.Context, id string) error { + return s.waitSnapshotOn(ctx, s.prodAPI(), id) +} + +func (s *Scanner) waitLocalSnapshot(ctx context.Context, id string) error { + return s.waitSnapshotOn(ctx, s.API, id) +} + +func (s *Scanner) waitSnapshotOn(ctx context.Context, api ec2API, id string) error { return s.poll(ctx, func() (bool, error) { - out, err := s.API.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{SnapshotIds: []string{id}}) + out, err := api.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{SnapshotIds: []string{id}}) if err != nil { return false, fmt.Errorf("describe snapshot: %w", err) } @@ -411,9 +467,20 @@ func (s *Scanner) deleteVolume(volID string) { } func (s *Scanner) deleteSnapshot(snapID string) { + s.deleteSnapshotOn(s.prodAPI(), snapID) +} + +func (s *Scanner) deleteLocalSnapshot(snapID string) { + s.deleteSnapshotOn(s.API, snapID) +} + +func (s *Scanner) deleteSnapshotOn(api ec2API, snapID string) { + if snapID == "" || api == nil { + return + } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - _, err := s.API.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapID)}) + _, err := api.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapID)}) if isNotFound(err) { return } diff --git a/internal/workload/aws/scan_test.go b/internal/workload/aws/scan_test.go index 58231f0..e444b04 100644 --- a/internal/workload/aws/scan_test.go +++ b/internal/workload/aws/scan_test.go @@ -23,14 +23,21 @@ import ( func TestPermissionsCoverSnapshotLifecycle(t *testing.T) { for _, action := range []string{ "ec2:CreateSnapshot", - "ec2:AttachVolume", + "ec2:ModifySnapshotAttribute", "ec2:DeleteSnapshot", - "ec2:DeleteVolume", "kms:CreateGrant", "kms:Decrypt", } { if !slices.Contains(Permissions, action) { - t.Errorf("missing %s", action) + t.Errorf("production role missing %s", action) + } + } + for _, action := range []string{"ec2:AttachVolume", "ec2:CreateVolume", "ec2:CopySnapshot", "ec2:DeleteVolume"} { + if slices.Contains(Permissions, action) { + t.Errorf("production role grants %s", action) + } + if !slices.Contains(WorkerPermissions, action) { + t.Errorf("scanner role missing %s", action) } } } @@ -212,6 +219,78 @@ func TestScanPaginatesInstances(t *testing.T) { } } +func TestScanSharesSnapshotAndMountsTheCopy(t *testing.T) { + prod := newFake([]ec2types.Instance{linuxInstance("i-web", "web", "vol-root")}) + local := newFake([]ec2types.Instance{testInstances()[0]}) + scanner := testScanner(t, local, func(ctx context.Context, volumeID string) (string, func() error, error) { + dir := t.TempDir() + writeRoot(t, dir) + return dir, func() error { return nil }, nil + }, workload.Walk) + scanner.Prod = prod + scanner.AccountID = "999988887777" + scanner.Worker.Region = "us-east-1" + scanner.Only = []string{"i-web"} + + var got []Result + if err := scanner.Scan(context.Background(), func(res Result) error { + got = append(got, res) + return nil + }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Skipped != "" || len(got[0].Packages) == 0 { + t.Fatalf("results = %#v", got) + } + assertOrder(t, prod.log, "CreateSnapshot", "ModifySnapshotAttribute", "DeleteSnapshot") + assertOrder(t, local.log, "CopySnapshot", "CreateVolume", "AttachVolume", "DetachVolume", "DeleteVolume", "DeleteSnapshot") + if slices.Contains(prod.log, "AttachVolume") || slices.Contains(prod.log, "CreateVolume") || slices.Contains(prod.log, "CopySnapshot") { + t.Fatalf("production account mounted a volume: %v", prod.log) + } + if len(local.volInputs) != 1 || aws.ToString(local.volInputs[0].SnapshotId) != "snap-1" { + t.Fatalf("volume = %#v", local.volInputs) + } + if len(local.copyInputs) != 1 || aws.ToString(local.copyInputs[0].SourceSnapshotId) != "snap-1" || aws.ToString(local.copyInputs[0].SourceRegion) != "us-east-1" { + t.Fatalf("copy = %#v", local.copyInputs) + } + if len(prod.shareInputs) != 2 || aws.ToString(prod.shareInputs[0].SnapshotId) != "snap-1" { + t.Fatalf("share = %#v", prod.shareInputs) + } + if len(prod.shareInputs[0].CreateVolumePermission.Add) != 1 || aws.ToString(prod.shareInputs[0].CreateVolumePermission.Add[0].UserId) != "999988887777" { + t.Fatalf("share add = %#v", prod.shareInputs[0].CreateVolumePermission) + } + if len(prod.shareInputs[1].CreateVolumePermission.Remove) != 1 { + t.Fatalf("share remove = %#v", prod.shareInputs[1].CreateVolumePermission) + } + if len(prod.snaps) != 0 || len(local.snaps) != 0 || len(local.vols) != 0 { + t.Fatalf("leftover prod snaps=%d local snaps=%d vols=%d", len(prod.snaps), len(local.snaps), len(local.vols)) + } +} + +func TestScanRefusesEncryptedSnapshotWithoutScannerKey(t *testing.T) { + prod := newFake([]ec2types.Instance{linuxInstance("i-web", "web", "vol-root")}) + prod.encryptSnapshots = true + local := newFake([]ec2types.Instance{testInstances()[0]}) + scanner := testScanner(t, local, func(context.Context, string) (string, func() error, error) { + t.Fatal("encrypted snapshot was mounted") + return "", nil, nil + }, workload.Walk) + scanner.Prod = prod + scanner.AccountID = "999988887777" + scanner.Worker.Region = "us-east-1" + scanner.Only = []string{"i-web"} + err := scanner.Scan(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "--kms-key") { + t.Fatalf("err = %v", err) + } + if slices.Contains(local.log, "CopySnapshot") || slices.Contains(local.log, "CreateVolume") { + t.Fatalf("local log = %v", local.log) + } + if len(prod.snaps) != 0 { + t.Fatal("source snapshot left behind") + } +} + func TestFreeDeviceSkipsNamesInUse(t *testing.T) { got := freeDevice([]ec2types.InstanceBlockDeviceMapping{ {DeviceName: aws.String("/dev/xvdf")}, @@ -327,23 +406,26 @@ type volRec struct { } type fakeEC2 struct { - instances []ec2types.Instance - pages int - describePages int - log []string - snapN int - volN int - snaps map[string]*snapState - vols map[string]*volRec - snapInputs []*ec2.CreateSnapshotInput - volInputs []*ec2.CreateVolumeInput - attachInputs []*ec2.AttachVolumeInput - stuckSnapshot bool - delaySnapshot bool - images []ec2types.Image - shared []ec2types.Image - imageErr map[string]error - denySnap map[string]error + instances []ec2types.Instance + pages int + describePages int + log []string + snapN int + volN int + snaps map[string]*snapState + vols map[string]*volRec + snapInputs []*ec2.CreateSnapshotInput + volInputs []*ec2.CreateVolumeInput + attachInputs []*ec2.AttachVolumeInput + stuckSnapshot bool + delaySnapshot bool + images []ec2types.Image + shared []ec2types.Image + imageErr map[string]error + denySnap map[string]error + encryptSnapshots bool + shareInputs []*ec2.ModifySnapshotAttributeInput + copyInputs []*ec2.CopySnapshotInput } func newFake(instances []ec2types.Instance) *fakeEC2 { @@ -407,10 +489,26 @@ func (f *fakeEC2) DescribeSnapshots(_ context.Context, in *ec2.DescribeSnapshots Snapshots: []ec2types.Snapshot{{ SnapshotId: aws.String(in.SnapshotIds[0]), State: st.state, + Encrypted: aws.Bool(f.encryptSnapshots), }}, }, nil } +func (f *fakeEC2) ModifySnapshotAttribute(_ context.Context, in *ec2.ModifySnapshotAttributeInput, _ ...func(*ec2.Options)) (*ec2.ModifySnapshotAttributeOutput, error) { + f.log = append(f.log, "ModifySnapshotAttribute") + f.shareInputs = append(f.shareInputs, in) + return &ec2.ModifySnapshotAttributeOutput{}, nil +} + +func (f *fakeEC2) CopySnapshot(_ context.Context, in *ec2.CopySnapshotInput, _ ...func(*ec2.Options)) (*ec2.CopySnapshotOutput, error) { + f.log = append(f.log, "CopySnapshot") + f.copyInputs = append(f.copyInputs, in) + f.snapN++ + id := fmt.Sprintf("snap-%d", f.snapN) + f.snaps[id] = &snapState{state: ec2types.SnapshotStateCompleted} + return &ec2.CopySnapshotOutput{SnapshotId: aws.String(id)}, nil +} + func (f *fakeEC2) DescribeImages(_ context.Context, in *ec2.DescribeImagesInput, _ ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error) { f.log = append(f.log, "DescribeImages") if in == nil { diff --git a/internal/workload/azure/client.go b/internal/workload/azure/client.go index ce2ae4f..16cf59b 100644 --- a/internal/workload/azure/client.go +++ b/internal/workload/azure/client.go @@ -102,7 +102,11 @@ func (c *Client) DeleteSnapshot(ctx context.Context, group, name string) error { func (c *Client) CreateDisk(ctx context.Context, group, location, snapshotName string, zones []string, vm string) (string, error) { name := objectName(vm) - poller, err := c.disks.BeginCreateOrUpdate(ctx, group, name, diskBody(location, c.resourceID(group, "snapshots", snapshotName), zones, vm), nil) + source := snapshotName + if !strings.HasPrefix(snapshotName, "/") { + source = c.resourceID(group, "snapshots", snapshotName) + } + poller, err := c.disks.BeginCreateOrUpdate(ctx, group, name, diskBody(location, source, zones, vm), nil) if err != nil { return "", fmt.Errorf("create disk: %w", err) } diff --git a/internal/workload/azure/client_test.go b/internal/workload/azure/client_test.go index ca35380..a49d413 100644 --- a/internal/workload/azure/client_test.go +++ b/internal/workload/azure/client_test.go @@ -94,9 +94,11 @@ func TestCheckSubscription(t *testing.T) { if err := CheckSubscription("sub", "SUB"); err != nil { t.Fatal(err) } - err := CheckSubscription("other", "sub") - if err == nil || err.Error() != "this VM is in subscription other; set AZURE_SUBSCRIPTION_ID=other" { - t.Fatalf("err = %v", err) + if err := CheckSubscription("other", "sub"); err != nil { + t.Fatal(err) + } + if err := CheckSubscription("", "sub"); err == nil { + t.Fatal("expected an empty subscription to fail") } } diff --git a/internal/workload/azure/identity.go b/internal/workload/azure/identity.go index 536673f..f351183 100644 --- a/internal/workload/azure/identity.go +++ b/internal/workload/azure/identity.go @@ -25,7 +25,7 @@ var metadataClient = &http.Client{ func LookupWorker(ctx context.Context) (Worker, error) { body, err := fetchMetadata(ctx) if err != nil { - return Worker{}, fmt.Errorf("om scan workload azure runs on an Azure VM in this subscription: %w", err) + return Worker{}, fmt.Errorf("om scan workload azure runs on an Azure VM in the scanner subscription: %w", err) } worker, err := parseMetadata(body) if err != nil { @@ -81,13 +81,11 @@ func parseMetadata(body []byte) (Worker, error) { return worker, nil } -// CheckSubscription reports when this VM is in a different subscription than AZURE_SUBSCRIPTION_ID. +// CheckSubscription reports when either subscription id is empty. +// The worker may run in a different subscription than AZURE_SUBSCRIPTION_ID. func CheckSubscription(workerSub, configured string) error { if workerSub == "" || configured == "" { return fmt.Errorf("Azure subscription is empty") } - if !strings.EqualFold(workerSub, configured) { - return fmt.Errorf("this VM is in subscription %s; set AZURE_SUBSCRIPTION_ID=%s", workerSub, workerSub) - } return nil } diff --git a/internal/workload/azure/image.go b/internal/workload/azure/image.go index ac4d422..0f1aa8f 100644 --- a/internal/workload/azure/image.go +++ b/internal/workload/azure/image.go @@ -17,15 +17,13 @@ import ( "github.com/OpenSourceOM/core/internal/workload" ) -// ImagePermissions are the Azure RBAC actions om scan workload image azure adds. -// The image itself is not deleted. om scan azure does not gain these actions. +// ImagePermissions are the Azure RBAC actions granted on the subscription +// being scanned so the worker can read boot images. Disk create and attach +// belong to WorkerPermissions. The image itself is not deleted. var ImagePermissions = []string{ "Microsoft.Compute/virtualMachines/read", "Microsoft.Compute/images/read", "Microsoft.Compute/galleries/images/versions/read", - "Microsoft.Compute/disks/write", - "Microsoft.Compute/disks/delete", - "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", } // ImageVM is a virtual machine launched from one boot image. @@ -67,8 +65,11 @@ type ImageAPI interface { var _ ImageAPI = (*Client)(nil) // ImageScanner reads each boot image once and deletes the disk it created. +// Disk, when set, is the scanner subscription. API lists images in the +// subscription being scanned. type ImageScanner struct { API ImageAPI + Disk ImageAPI Worker Worker Only []string Timeout time.Duration @@ -88,7 +89,7 @@ func (s *ImageScanner) Scan(ctx context.Context, write func(ImageResult) error) if s.API == nil || s.Mount == nil || s.Walk == nil { return fmt.Errorf("workload scan is missing an Azure client, mount, or package walker") } - images, luns, err := s.API.ListBootImages(ctx, s.Worker) + images, luns, err := s.bootImages(ctx) if err != nil { return err } @@ -132,7 +133,7 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, luns []int3 s.OnStart(img.Key, img.Name) } group := s.Worker.ResourceGroup - disk, err := s.API.CreateDiskFromImage(ctx, group, normalizeLocation(s.Worker.Location), img, s.Worker.zones()) + disk, err := s.diskAPI().CreateDiskFromImage(ctx, group, normalizeLocation(s.Worker.Location), img, s.Worker.zones()) if disk != "" { defer s.deleteDisk(group, disk) } @@ -143,7 +144,7 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, luns []int3 } return ImageResult{}, fmt.Errorf("%s: %w", imageLabel(img), err) } - if err := s.API.Attach(ctx, group, s.Worker.Name, disk, lun); err != nil { + if err := s.diskAPI().Attach(ctx, group, s.Worker.Name, disk, lun); err != nil { return ImageResult{}, fmt.Errorf("%s: %w", imageLabel(img), err) } defer s.detach(group, disk) @@ -170,10 +171,40 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, luns []int3 return res, nil } +type imageCatalog interface { + ListImages(ctx context.Context, location string) ([]BootImage, error) +} + +func (s *ImageScanner) diskAPI() ImageAPI { + if s.Disk != nil { + return s.Disk + } + return s.API +} + +func (s *ImageScanner) bootImages(ctx context.Context) ([]BootImage, []int32, error) { + if s.Disk == nil { + return s.API.ListBootImages(ctx, s.Worker) + } + catalog, ok := s.API.(imageCatalog) + if !ok { + return nil, nil, fmt.Errorf("production client cannot list images") + } + images, err := catalog.ListImages(ctx, s.Worker.Location) + if err != nil { + return nil, nil, err + } + _, luns, err := s.Disk.ListBootImages(ctx, s.Worker) + if err != nil { + return nil, nil, err + } + return images, luns, nil +} + func (s *ImageScanner) detach(group, disk string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.Detach(ctx, group, s.Worker.Name, disk); err != nil && !isNotFound(err) { + if err := s.diskAPI().Detach(ctx, group, s.Worker.Name, disk); err != nil && !isNotFound(err) { s.note(fmt.Errorf("detach disk %s: %w", disk, err)) } } @@ -181,7 +212,7 @@ func (s *ImageScanner) detach(group, disk string) { func (s *ImageScanner) deleteDisk(group, name string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.DeleteDisk(ctx, group, name); err != nil && !isNotFound(err) { + if err := s.diskAPI().DeleteDisk(ctx, group, name); err != nil && !isNotFound(err) { s.note(fmt.Errorf("delete disk %s: %w", name, err)) } } @@ -240,6 +271,19 @@ func imageLabel(img BootImage) string { // ListBootImages returns managed images in the worker's region, images that // virtual machines there were launched from, and the worker's data-disk LUNs. +func (c *Client) ListImages(ctx context.Context, location string) ([]BootImage, error) { + location = normalizeLocation(location) + machines, err := c.List(ctx, location) + if err != nil { + return nil, err + } + listed, err := c.listManagedImages(ctx) + if err != nil { + return nil, err + } + return assembleBootImages(location, c.sub, machines, listed), nil +} + func (c *Client) ListBootImages(ctx context.Context, worker Worker) ([]BootImage, []int32, error) { if worker.Name == "" || worker.ResourceGroup == "" || worker.Location == "" { return nil, nil, fmt.Errorf("worker name, resource group, and location are required") diff --git a/internal/workload/azure/image_test.go b/internal/workload/azure/image_test.go index 59119b3..9e69353 100644 --- a/internal/workload/azure/image_test.go +++ b/internal/workload/azure/image_test.go @@ -41,12 +41,15 @@ func TestMachineFromImageReference(t *testing.T) { } func TestImagePermissionsDoNotSnapshot(t *testing.T) { - if !slices.Contains(ImagePermissions, "Microsoft.Compute/images/read") || !slices.Contains(ImagePermissions, "Microsoft.Compute/disks/write") { + if !slices.Contains(ImagePermissions, "Microsoft.Compute/images/read") { t.Fatalf("permissions = %v", ImagePermissions) } + if !slices.Contains(WorkerPermissions, "Microsoft.Compute/disks/write") { + t.Fatalf("worker = %v", WorkerPermissions) + } for _, action := range ImagePermissions { - if strings.Contains(action, "snapshots") { - t.Errorf("image scan must not use %s", action) + if strings.Contains(action, "snapshots") || strings.Contains(action, "disks/write") || strings.Contains(action, "attachDetach") { + t.Errorf("image scan must not use %s on the production subscription", action) } } } diff --git a/internal/workload/azure/scan.go b/internal/workload/azure/scan.go index ddd7532..be1aba0 100644 --- a/internal/workload/azure/scan.go +++ b/internal/workload/azure/scan.go @@ -13,18 +13,34 @@ import ( "github.com/OpenSourceOM/core/internal/workload" ) -// Permissions are the Azure RBAC actions om scan workload azure adds. -// keys/unwrap/action and keys/wrap/action are the grant a customer-managed key needs. -// om scan azure does not gain these actions. +// Permissions are the Azure RBAC actions granted on the subscription being +// scanned. That identity creates and deletes snapshots. It does not attach a +// disk. keys/unwrap/action and keys/wrap/action are the grant a +// customer-managed key needs. om scan azure does not gain these actions. var Permissions = []string{ "Microsoft.Compute/virtualMachines/read", "Microsoft.Compute/disks/read", - "Microsoft.Compute/disks/beginGetAccess/action", "Microsoft.Compute/snapshots/read", "Microsoft.Compute/snapshots/write", "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/images/read", + "Microsoft.Compute/galleries/images/versions/read", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.KeyVault/vaults/keys/wrap/action", + "Microsoft.KeyVault/vaults/keys/unwrap/action", +} + +// WorkerPermissions are the Azure RBAC actions granted in the scanner +// subscription, where the worker attaches the disk. snapshots/read lets that +// identity create a disk from a snapshot in the subscription being scanned. +var WorkerPermissions = []string{ + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/disks/read", "Microsoft.Compute/disks/write", "Microsoft.Compute/disks/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/images/read", + "Microsoft.Compute/galleries/images/versions/read", "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", "Microsoft.Compute/diskEncryptionSets/read", "Microsoft.KeyVault/vaults/keys/wrap/action", @@ -82,14 +98,20 @@ type API interface { } // Scanner snapshots Linux OS disks, reads them, and deletes what it created. +// API is the subscription being scanned. Disk, when set, is the scanner +// subscription where the worker attaches the disk. ProdSubscription and +// SnapshotGroup name the snapshot that Disk creates a managed disk from. type Scanner struct { - API API - Worker Worker - Only []string - Timeout time.Duration - Mount func(ctx context.Context, lun string) (root string, cleanup func() error, err error) - Walk func(root string) ([]string, error) - OnStart func(name, group string) + API API + Disk API + ProdSubscription string + SnapshotGroup string + Worker Worker + Only []string + Timeout time.Duration + Mount func(ctx context.Context, lun string) (root string, cleanup func() error, err error) + Walk func(root string) ([]string, error) + OnStart func(name, group string) cleanupErrs []error } @@ -134,9 +156,21 @@ func (s *Scanner) list(ctx context.Context) ([]Machine, Machine, error) { if err != nil { return nil, Machine{}, err } - worker, err := findMachine(machines, s.Worker.ResourceGroup+"/"+s.Worker.Name) - if err != nil { - return nil, Machine{}, fmt.Errorf("worker %s/%s is not in %s", s.Worker.ResourceGroup, s.Worker.Name, normalizeLocation(s.Worker.Location)) + var worker Machine + if s.Disk != nil { + scanned, err := s.Disk.List(ctx, normalizeLocation(s.Worker.Location)) + if err != nil { + return nil, Machine{}, err + } + worker, err = findMachine(scanned, s.Worker.ResourceGroup+"/"+s.Worker.Name) + if err != nil { + return nil, Machine{}, fmt.Errorf("worker %s/%s is not in the scanner subscription", s.Worker.ResourceGroup, s.Worker.Name) + } + } else { + worker, err = findMachine(machines, s.Worker.ResourceGroup+"/"+s.Worker.Name) + if err != nil { + return nil, Machine{}, fmt.Errorf("worker %s/%s is not in %s", s.Worker.ResourceGroup, s.Worker.Name, normalizeLocation(s.Worker.Location)) + } } if len(s.Only) == 0 { return machines, worker, nil @@ -166,7 +200,7 @@ func (s *Scanner) scanVM(ctx context.Context, vm Machine, luns []int32) (Result, s.OnStart(vm.Name, vm.ResourceGroup) } - group := s.Worker.ResourceGroup + group := s.snapshotGroup() snap, err := s.API.CreateSnapshot(ctx, group, normalizeLocation(s.Worker.Location), vm.OSDiskID, vm.Name) if snap != "" { defer s.deleteSnapshot(group, snap) @@ -174,17 +208,18 @@ func (s *Scanner) scanVM(ctx context.Context, vm Machine, luns []int32) (Result, if err != nil { return Result{}, fmt.Errorf("%s: %w", vmLabel(res), err) } - disk, err := s.API.CreateDisk(ctx, group, normalizeLocation(s.Worker.Location), snap, s.Worker.zones(), vm.Name) + diskGroup := s.Worker.ResourceGroup + disk, err := s.diskAPI().CreateDisk(ctx, diskGroup, normalizeLocation(s.Worker.Location), s.snapshotSource(group, snap), s.Worker.zones(), vm.Name) if disk != "" { - defer s.deleteDisk(group, disk) + defer s.deleteDisk(diskGroup, disk) } if err != nil { return Result{}, fmt.Errorf("%s: %w", vmLabel(res), err) } - if err := s.API.Attach(ctx, group, s.Worker.Name, disk, lun); err != nil { + if err := s.diskAPI().Attach(ctx, diskGroup, s.Worker.Name, disk, lun); err != nil { return Result{}, fmt.Errorf("%s: %w", vmLabel(res), err) } - defer s.detach(group, disk) + defer s.detach(diskGroup, disk) root, unmount, err := s.Mount(ctx, fmt.Sprintf("%d", lun)) if err != nil { @@ -215,10 +250,31 @@ func (w Worker) zones() []string { return []string{w.Zone} } +func (s *Scanner) diskAPI() API { + if s.Disk != nil { + return s.Disk + } + return s.API +} + +func (s *Scanner) snapshotGroup() string { + if s.SnapshotGroup != "" { + return s.SnapshotGroup + } + return s.Worker.ResourceGroup +} + +func (s *Scanner) snapshotSource(group, name string) string { + if s.Disk == nil || s.ProdSubscription == "" || strings.HasPrefix(name, "/") { + return name + } + return "/subscriptions/" + s.ProdSubscription + "/resourceGroups/" + group + "/providers/Microsoft.Compute/snapshots/" + name +} + func (s *Scanner) detach(group, disk string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.Detach(ctx, group, s.Worker.Name, disk); err != nil && !isNotFound(err) { + if err := s.diskAPI().Detach(ctx, group, s.Worker.Name, disk); err != nil && !isNotFound(err) { s.note(fmt.Errorf("detach disk %s: %w", disk, err)) } } @@ -226,7 +282,7 @@ func (s *Scanner) detach(group, disk string) { func (s *Scanner) deleteDisk(group, name string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.DeleteDisk(ctx, group, name); err != nil && !isNotFound(err) { + if err := s.diskAPI().DeleteDisk(ctx, group, name); err != nil && !isNotFound(err) { s.note(fmt.Errorf("delete disk %s: %w", name, err)) } } diff --git a/internal/workload/azure/scan_test.go b/internal/workload/azure/scan_test.go index e0aa091..11190fd 100644 --- a/internal/workload/azure/scan_test.go +++ b/internal/workload/azure/scan_test.go @@ -18,13 +18,23 @@ import ( func TestPermissionsCoverSnapshotLifecycle(t *testing.T) { for _, action := range []string{ "Microsoft.Compute/snapshots/write", - "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", "Microsoft.Compute/snapshots/delete", - "Microsoft.Compute/disks/delete", "Microsoft.KeyVault/vaults/keys/unwrap/action", } { if !slices.Contains(Permissions, action) { - t.Errorf("missing %s", action) + t.Errorf("production role missing %s", action) + } + } + if slices.Contains(Permissions, "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action") || slices.Contains(Permissions, "Microsoft.Compute/disks/write") { + t.Fatalf("production role attaches disks: %v", Permissions) + } + for _, action := range []string{ + "Microsoft.Compute/virtualMachines/attachDetachDataDisks/action", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/disks/delete", + } { + if !slices.Contains(WorkerPermissions, action) { + t.Errorf("scanner role missing %s", action) } } } diff --git a/internal/workload/gcp/client.go b/internal/workload/gcp/client.go index 7983b65..cf768bd 100644 --- a/internal/workload/gcp/client.go +++ b/internal/workload/gcp/client.go @@ -99,7 +99,11 @@ func (c *Client) DeleteSnapshot(ctx context.Context, name string) error { func (c *Client) CreateDisk(ctx context.Context, zone, snapshotName, vm string) (string, error) { name := objectName(vm) - op, err := c.svc.Disks.Insert(c.project, zone, diskBody(name, zone, c.snapshotLink(snapshotName), vm)).Context(ctx).Do() + link := snapshotName + if !strings.Contains(snapshotName, "/") { + link = c.snapshotLink(snapshotName) + } + op, err := c.svc.Disks.Insert(c.project, zone, diskBody(name, zone, link, vm)).Context(ctx).Do() if err != nil { return "", fmt.Errorf("create disk: %w", err) } diff --git a/internal/workload/gcp/identity.go b/internal/workload/gcp/identity.go index 0119074..774ac02 100644 --- a/internal/workload/gcp/identity.go +++ b/internal/workload/gcp/identity.go @@ -35,7 +35,7 @@ func LookupWorker(ctx context.Context) (Worker, error) { func lookupWorker(ctx context.Context, meta metaAPI) (Worker, error) { if !meta.OnGCE(ctx) { - return Worker{}, fmt.Errorf("om scan workload gcp runs on a GCE instance in this project") + return Worker{}, fmt.Errorf("om scan workload gcp runs on a GCE instance in the scanner project") } project, err := meta.ProjectID(ctx) if err != nil { @@ -56,13 +56,11 @@ func lookupWorker(ctx context.Context, meta metaAPI) (Worker, error) { return Worker{Name: name, ProjectID: project, Zone: zone}, nil } -// CheckProject reports when this instance is in a different project than GCP_PROJECT_ID. +// CheckProject reports when either project id is empty. +// The worker may run in a different project than GCP_PROJECT_ID. func CheckProject(workerProject, configured string) error { if workerProject == "" || configured == "" { return fmt.Errorf("GCP project is empty") } - if workerProject != configured { - return fmt.Errorf("this instance is in project %s; set GCP_PROJECT_ID=%s", workerProject, workerProject) - } return nil } diff --git a/internal/workload/gcp/image.go b/internal/workload/gcp/image.go index 856d980..dcfef07 100644 --- a/internal/workload/gcp/image.go +++ b/internal/workload/gcp/image.go @@ -17,23 +17,15 @@ import ( "google.golang.org/api/googleapi" ) -// ImagePermissions are the IAM permissions om scan workload image gcp adds. -// The image itself is not deleted. useToDecrypt is the unwrap a -// customer-managed key needs when the disk keeps that key. om scan gcp does -// not gain these permissions. +// ImagePermissions are the IAM permissions granted on the project being +// scanned so the worker can read boot images. Disk create and attach belong +// to WorkerPermissions. The image itself is not deleted. var ImagePermissions = []string{ "compute.instances.list", "compute.disks.list", "compute.images.list", + "compute.images.get", "compute.images.useReadOnly", - "compute.disks.create", - "compute.disks.delete", - "compute.disks.useReadOnly", - "compute.instances.attachDisk", - "compute.instances.detachDisk", - "compute.zoneOperations.get", - "cloudkms.cryptoKeyVersions.useToDecrypt", - "cloudkms.cryptoKeyVersions.useToEncrypt", } // ImageVM is a GCE instance launched from one boot image. @@ -73,8 +65,11 @@ type ImageAPI interface { var _ ImageAPI = (*Client)(nil) // ImageScanner reads each boot image once and deletes the disk it created. +// Disk, when set, is the scanner project. API lists images in the project +// being scanned. type ImageScanner struct { API ImageAPI + Disk ImageAPI Worker Worker Only []string Timeout time.Duration @@ -95,7 +90,7 @@ func (s *ImageScanner) Scan(ctx context.Context, write func(ImageResult) error) if s.API == nil || s.Mount == nil || s.Walk == nil { return fmt.Errorf("workload scan is missing a Compute client, mount, or package walker") } - images, devices, err := s.API.ListBootImages(ctx, s.Worker) + images, devices, err := s.bootImages(ctx) if err != nil { return err } @@ -138,7 +133,7 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, devices []s if s.OnStart != nil { s.OnStart(img.Link, img.Name) } - disk, err := s.API.CreateDiskFromImage(ctx, s.Worker.Zone, img.Link, img.Name) + disk, err := s.diskAPI().CreateDiskFromImage(ctx, s.Worker.Zone, img.Link, img.Name) if disk != "" { defer s.deleteDisk(s.Worker.Zone, disk) } @@ -149,7 +144,7 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, devices []s } return ImageResult{}, fmt.Errorf("%s: %w", imageLabel(img), err) } - if err := s.API.Attach(ctx, s.Worker.Zone, s.Worker.Name, disk, device); err != nil { + if err := s.diskAPI().Attach(ctx, s.Worker.Zone, s.Worker.Name, disk, device); err != nil { return ImageResult{}, fmt.Errorf("%s: %w", imageLabel(img), err) } defer s.detach(s.Worker.Zone, device) @@ -176,10 +171,40 @@ func (s *ImageScanner) scanImage(ctx context.Context, img BootImage, devices []s return res, nil } +type imageCatalog interface { + ListImages(ctx context.Context, region string) ([]BootImage, error) +} + +func (s *ImageScanner) diskAPI() ImageAPI { + if s.Disk != nil { + return s.Disk + } + return s.API +} + +func (s *ImageScanner) bootImages(ctx context.Context) ([]BootImage, []string, error) { + if s.Disk == nil { + return s.API.ListBootImages(ctx, s.Worker) + } + catalog, ok := s.API.(imageCatalog) + if !ok { + return nil, nil, fmt.Errorf("production client cannot list images") + } + images, err := catalog.ListImages(ctx, s.Worker.Region()) + if err != nil { + return nil, nil, err + } + _, devices, err := s.Disk.ListBootImages(ctx, s.Worker) + if err != nil { + return nil, nil, err + } + return images, devices, nil +} + func (s *ImageScanner) detach(zone, device string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.Detach(ctx, zone, s.Worker.Name, device); err != nil && !isNotFound(err) { + if err := s.diskAPI().Detach(ctx, zone, s.Worker.Name, device); err != nil && !isNotFound(err) { s.note(fmt.Errorf("detach disk %s: %w", device, err)) } } @@ -187,7 +212,7 @@ func (s *ImageScanner) detach(zone, device string) { func (s *ImageScanner) deleteDisk(zone, name string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.DeleteDisk(ctx, zone, name); err != nil && !isNotFound(err) { + if err := s.diskAPI().DeleteDisk(ctx, zone, name); err != nil && !isNotFound(err) { s.note(fmt.Errorf("delete disk %s: %w", name, err)) } } @@ -246,6 +271,25 @@ func imageLabel(img BootImage) string { // ListBootImages returns project images and images that instances in the // worker's region were launched from, plus the worker's device names. +func (c *Client) ListImages(ctx context.Context, region string) ([]BootImage, error) { + instances, err := c.List(ctx, region) + if err != nil { + return nil, err + } + sources, err := c.diskImages(ctx, region) + if err != nil { + return nil, err + } + for i := range instances { + instances[i].SourceImage = sources[imageKey(instances[i].BootSource)] + } + listed, err := c.projectImages(ctx) + if err != nil { + return nil, err + } + return assembleGCE(listed, instances), nil +} + func (c *Client) ListBootImages(ctx context.Context, worker Worker) ([]BootImage, []string, error) { if worker.Name == "" || worker.Zone == "" { return nil, nil, fmt.Errorf("worker name and zone are required") diff --git a/internal/workload/gcp/image_test.go b/internal/workload/gcp/image_test.go index c7a0aa5..b7d7207 100644 --- a/internal/workload/gcp/image_test.go +++ b/internal/workload/gcp/image_test.go @@ -15,12 +15,15 @@ import ( ) func TestImagePermissionsDoNotSnapshot(t *testing.T) { - if !slices.Contains(ImagePermissions, "compute.images.useReadOnly") || !slices.Contains(ImagePermissions, "compute.disks.create") { + if !slices.Contains(ImagePermissions, "compute.images.useReadOnly") { t.Fatalf("permissions = %v", ImagePermissions) } + if !slices.Contains(WorkerPermissions, "compute.disks.create") { + t.Fatalf("worker = %v", WorkerPermissions) + } for _, action := range ImagePermissions { - if strings.Contains(action, "snapshot") { - t.Errorf("image scan must not use %s", action) + if strings.Contains(action, "snapshot") || action == "compute.disks.create" || action == "compute.instances.attachDisk" { + t.Errorf("image scan must not use %s on the production project", action) } } } diff --git a/internal/workload/gcp/scan.go b/internal/workload/gcp/scan.go index f17e9a8..618d5c9 100644 --- a/internal/workload/gcp/scan.go +++ b/internal/workload/gcp/scan.go @@ -13,18 +13,36 @@ import ( "github.com/OpenSourceOM/core/internal/workload" ) -// Permissions are the IAM permissions om scan workload gcp adds. -// useToDecrypt is the unwrap a customer-managed key needs. useToEncrypt is the -// matching wrap when the snapshot keeps that key. om scan gcp does not gain these. +// Permissions are the IAM permissions granted on the project being scanned. +// The scanner service account creates snapshots there and may use them to +// create a disk in the scanner project. It does not attach a disk in this +// project. useToDecrypt is the unwrap a customer-managed key needs. +// useToEncrypt is the wrap when the snapshot keeps that key. om scan gcp does +// not gain these permissions. var Permissions = []string{ "compute.instances.list", "compute.instances.get", + "compute.disks.list", "compute.disks.createSnapshot", "compute.regionDisks.createSnapshot", "compute.snapshots.create", "compute.snapshots.get", "compute.snapshots.delete", "compute.snapshots.useReadOnly", + "compute.images.list", + "compute.images.get", + "compute.images.useReadOnly", + "compute.zoneOperations.get", + "compute.globalOperations.get", + "cloudkms.cryptoKeyVersions.useToDecrypt", + "cloudkms.cryptoKeyVersions.useToEncrypt", +} + +// WorkerPermissions are the IAM permissions granted on the scanner project, +// where the worker attaches the disk. +var WorkerPermissions = []string{ + "compute.instances.list", + "compute.instances.get", "compute.disks.create", "compute.disks.get", "compute.disks.delete", @@ -81,14 +99,19 @@ type API interface { } // Scanner snapshots Linux boot disks, reads them, and deletes what it created. +// API is the project being scanned. Disk, when set, is the scanner project. +// ProdProject is the project id embedded in the snapshot URL the disk is +// created from. type Scanner struct { - API API - Worker Worker - Only []string - Timeout time.Duration - Mount func(ctx context.Context, device string) (root string, cleanup func() error, err error) - Walk func(root string) ([]string, error) - OnStart func(name, zone string) + API API + Disk API + ProdProject string + Worker Worker + Only []string + Timeout time.Duration + Mount func(ctx context.Context, device string) (root string, cleanup func() error, err error) + Walk func(root string) ([]string, error) + OnStart func(name, zone string) cleanupErrs []error } @@ -133,9 +156,21 @@ func (s *Scanner) list(ctx context.Context) ([]Instance, Instance, error) { if err != nil { return nil, Instance{}, err } - worker, err := findInstance(instances, s.Worker.Zone+"/"+s.Worker.Name) - if err != nil { - return nil, Instance{}, fmt.Errorf("worker %s/%s is not in %s", s.Worker.Zone, s.Worker.Name, s.Worker.Region()) + var worker Instance + if s.Disk != nil { + scanned, err := s.Disk.List(ctx, s.Worker.Region()) + if err != nil { + return nil, Instance{}, err + } + worker, err = findInstance(scanned, s.Worker.Zone+"/"+s.Worker.Name) + if err != nil { + return nil, Instance{}, fmt.Errorf("worker %s/%s is not in the scanner project", s.Worker.Zone, s.Worker.Name) + } + } else { + worker, err = findInstance(instances, s.Worker.Zone+"/"+s.Worker.Name) + if err != nil { + return nil, Instance{}, fmt.Errorf("worker %s/%s is not in %s", s.Worker.Zone, s.Worker.Name, s.Worker.Region()) + } } if len(s.Only) == 0 { return instances, worker, nil @@ -172,14 +207,14 @@ func (s *Scanner) scanInstance(ctx context.Context, inst Instance, devices []str if err != nil { return Result{}, fmt.Errorf("%s: %w", instLabel(res), err) } - disk, err := s.API.CreateDisk(ctx, s.Worker.Zone, snap, inst.Name) + disk, err := s.diskAPI().CreateDisk(ctx, s.Worker.Zone, s.snapshotRef(snap), inst.Name) if disk != "" { defer s.deleteDisk(s.Worker.Zone, disk) } if err != nil { return Result{}, fmt.Errorf("%s: %w", instLabel(res), err) } - if err := s.API.Attach(ctx, s.Worker.Zone, s.Worker.Name, disk, device); err != nil { + if err := s.diskAPI().Attach(ctx, s.Worker.Zone, s.Worker.Name, disk, device); err != nil { return Result{}, fmt.Errorf("%s: %w", instLabel(res), err) } defer s.detach(s.Worker.Zone, device) @@ -206,10 +241,24 @@ func (s *Scanner) scanInstance(ctx context.Context, inst Instance, devices []str return res, nil } +func (s *Scanner) diskAPI() API { + if s.Disk != nil { + return s.Disk + } + return s.API +} + +func (s *Scanner) snapshotRef(name string) string { + if s.Disk == nil || s.ProdProject == "" || strings.Contains(name, "/") { + return name + } + return "projects/" + s.ProdProject + "/global/snapshots/" + name +} + func (s *Scanner) detach(zone, device string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.Detach(ctx, zone, s.Worker.Name, device); err != nil && !isNotFound(err) { + if err := s.diskAPI().Detach(ctx, zone, s.Worker.Name, device); err != nil && !isNotFound(err) { s.note(fmt.Errorf("detach disk %s: %w", device, err)) } } @@ -217,7 +266,7 @@ func (s *Scanner) detach(zone, device string) { func (s *Scanner) deleteDisk(zone, name string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - if err := s.API.DeleteDisk(ctx, zone, name); err != nil && !isNotFound(err) { + if err := s.diskAPI().DeleteDisk(ctx, zone, name); err != nil && !isNotFound(err) { s.note(fmt.Errorf("delete disk %s: %w", name, err)) } } diff --git a/internal/workload/gcp/scan_test.go b/internal/workload/gcp/scan_test.go index f5981f7..e8633f9 100644 --- a/internal/workload/gcp/scan_test.go +++ b/internal/workload/gcp/scan_test.go @@ -19,13 +19,20 @@ import ( func TestPermissionsCoverSnapshotLifecycle(t *testing.T) { for _, action := range []string{ "compute.disks.createSnapshot", - "compute.instances.attachDisk", "compute.snapshots.delete", - "compute.disks.delete", + "compute.snapshots.useReadOnly", "cloudkms.cryptoKeyVersions.useToDecrypt", } { if !slices.Contains(Permissions, action) { - t.Errorf("missing %s", action) + t.Errorf("production role missing %s", action) + } + } + if slices.Contains(Permissions, "compute.instances.attachDisk") || slices.Contains(Permissions, "compute.disks.create") { + t.Fatalf("production role attaches disks: %v", Permissions) + } + for _, action := range []string{"compute.instances.attachDisk", "compute.disks.create", "compute.disks.delete"} { + if !slices.Contains(WorkerPermissions, action) { + t.Errorf("scanner role missing %s", action) } } } @@ -244,9 +251,11 @@ func TestCheckProject(t *testing.T) { if err := CheckProject("p", "p"); err != nil { t.Fatal(err) } - err := CheckProject("other", "p") - if err == nil || err.Error() != "this instance is in project other; set GCP_PROJECT_ID=other" { - t.Fatalf("err = %v", err) + if err := CheckProject("other", "p"); err != nil { + t.Fatal(err) + } + if err := CheckProject("", "p"); err == nil { + t.Fatal("expected an empty project to fail") } }