commit e88609d72426b2e3995f92b505525fd3175c468a Author: James Bland Date: Thu Nov 27 04:20:56 2025 -0500 feat: crud-pricing initial implementation Complete CRUD service for AWS pricing operations - single source of truth. Features: - Dual pricing model (retail + account-specific with auto EDP/PPA detection) - Get/Put pricing operations with intelligent caching - AWS Pricing API integration for public list prices - AWS Cost Explorer integration for account-specific pricing - Access counting for self-learning 14-day refresh - Query most-accessed instances (powers smart refresh) - TTL: 30 days (retail), 7 days (account-specific) Architecture: - All other lambdas use this for pricing operations - No direct DynamoDB access from other components - Consistent schema enforcement - Complete IAM setup for Pricing API, Cost Explorer, STS Infrastructure: - Complete Terraform configuration - Full CI/CD pipeline (Jenkinsfile) - Comprehensive documentation - Production-ready scaffolding Part of Phase 1 - foundation for pricing system. ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ebeffb --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/target +Cargo.lock +.env +*.swp +*.swo +*~ +.DS_Store diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f558f82 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "crud-pricing" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "bootstrap" +path = "src/main.rs" + +[dependencies] +# AWS SDK +aws-config = { version = "1.5", features = ["behavior-version-latest"] } +aws-sdk-dynamodb = "1.60" +aws-sdk-pricing = "1.60" +aws-sdk-costexplorer = "1.60" +aws-sdk-sts = "1.60" + +# Lambda runtime +lambda_runtime = "0.13" +tokio = { version = "1", features = ["full"] } + +# Serialization +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +# Date/Time +chrono = { version = "0.4", features = ["serde"] } + +# Logging +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } + +# Error handling +anyhow = "1.0" +thiserror = "1.0" + +[dev-dependencies] +tokio-test = "0.4" diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..378ac12 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,114 @@ +pipeline { + agent any + + options { + ansiColor('xterm') + disableConcurrentBuilds() + } + + stages { + stage('Setup') { + steps { + echo '๐Ÿ”ง Verifying Rust environment...' + sh ''' + rustc --version + cargo --version + ''' + } + } + + stage('Install Tools') { + steps { + echo '๐Ÿ“ฆ Installing test tools...' + sh ''' + if ! command -v cargo-nextest &> /dev/null; then + cargo install cargo-nextest + fi + if ! command -v cargo-llvm-cov &> /dev/null; then + cargo install cargo-llvm-cov + fi + if ! command -v cargo-audit &> /dev/null; then + cargo install cargo-audit + fi + rustup component add llvm-tools-preview + ''' + } + } + + stage('Security Audit') { + steps { + echo '๐Ÿ”’ Running cargo audit...' + sh ''' + cargo audit --json > audit-report.json || true + ''' + } + } + + stage('Check Code') { + steps { + echo '๐Ÿ” Running cargo check...' + sh ''' + cargo check --message-format json > cargo-check.log || true + ''' + } + } + + stage('Run Tests with Coverage') { + steps { + echo '๐Ÿงช Running tests with coverage...' + sh ''' + # Run tests with nextest and generate JUnit + cargo nextest run --profile ci --verbose || true + + # Create coverage directory and generate coverage + mkdir -p coverage + cargo llvm-cov nextest --cobertura --output-path coverage/cobertura.xml || true + ''' + } + } + + stage('Generate Docs') { + steps { + echo '๐Ÿ“š Generating documentation...' + sh ''' + cargo doc --no-deps || true + ''' + } + } + + stage('Publish Results') { + steps { + echo '๐Ÿ“‹ Publishing test results, coverage, and warnings...' + junit allowEmptyResults: true, testResults: 'target/nextest/ci/junit.xml' + recordCoverage(tools: [[parser: 'COBERTURA', pattern: '**/cobertura.xml']]) + recordIssues tool: cargo(pattern: 'cargo-check.log') + + echo '๐Ÿ“š Publishing documentation...' + publishHTML(target: [ + allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: 'target/doc', + reportFiles: 'crud_pricing/index.html', + reportName: 'Rust Documentation' + ]) + } + } + } + + post { + always { + echo '๐Ÿงน Archiving artifacts...' + archiveArtifacts artifacts: 'coverage/**/*.xml, target/nextest/**/*.xml, cargo-check.log, audit-report.json', allowEmptyArchive: true + } + success { + echo 'โœ… Build completed!' + } + unstable { + echo 'โš ๏ธ Build unstable - some tests may have failed' + } + failure { + echo 'โŒ Build failed!' + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ae110e0 --- /dev/null +++ b/README.md @@ -0,0 +1,664 @@ +# crud-pricing - AWS Pricing CRUD Operations + +**Single source of truth for all AWS pricing operations.** + +## What It Does + +This Lambda is the centralized CRUD service for AWS pricing data: +- โœ… **Get/Put pricing** in DynamoDB cache +- โœ… **Fetch from AWS Pricing API** (retail public prices) +- โœ… **Fetch from Cost Explorer** (account-specific with auto EDP/PPA!) +- โœ… **Access counting** for self-learning refresh +- โœ… **Query most-accessed instances** for smart 14-day refresh + +## Why This Exists + +**Problem:** Multiple lambdas touching pricing table = schema duplication + +**Solution:** Single CRUD Lambda pattern +- All pricing operations go through one place +- Consistent schema enforcement +- Single place to evolve pricing logic +- Reusable across all components + +## Architecture + +``` +tool-pricing-query (MCP tool) } +enrichment-server-pricing (Lambda) } โ†’ crud-pricing (CRUD) โ†’ DynamoDB +tool-pricing-refresh (Scheduler) } โ†“ + AWS Pricing API + AWS Cost Explorer +``` + +**Used by:** +- `tool-pricing-query` - MCP tool wrapper for agents +- `enrichment-server-pricing` - Server pricing enrichment +- `tool-pricing-refresh` - 14-day automated refresh + +**This Lambda is NOT registered with AgentCore** - it's internal CRUD only. + +## Operations + +### 1. Get - Retrieve pricing (with optional AWS fetch) + +```json +{ + "operation": { + "type": "get", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "fetchIfMissing": true + } +} +``` + +**Response:** +```json +{ + "pricing": { + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "ec2Pricing": { + "vcpus": 4, + "memoryGb": 16.0, + "onDemand": { + "hourly": 0.154, + "monthly": 112.42 + }, + "reserved": { + "standard": { + "1yr": { + "allUpfront": { + "effectiveMonthly": 72.27, + "totalUpfront": 867.24 + } + } + } + } + }, + "accessCount": 127, + "lastUpdated": "2025-11-27T00:00:00Z" + }, + "cacheStatus": "hit" +} +``` + +### 2. Put - Store pricing data + +```json +{ + "operation": { + "type": "put", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "pricingData": { /* PricingData object */ } + } +} +``` + +### 3. ListCommon - Get most-accessed instances + +```json +{ + "operation": { + "type": "listCommon", + "limit": 50, + "minAccessCount": 5 + } +} +``` + +**Response:** +```json +{ + "instances": [ + { + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "accessCount": 347, + "lastAccessed": "2025-11-27T10:30:00Z", + "lastUpdated": "2025-11-20T00:00:00Z" + } + ], + "count": 50 +} +``` + +**Used by tool-pricing-refresh to discover common instances!** + +### 4. IncrementAccess - Track usage + +```json +{ + "operation": { + "type": "incrementAccess", + "instanceType": "m6g.xlarge", + "region": "us-east-1" + } +} +``` + +### 5. QueryAwsApi - Direct AWS Pricing API call + +```json +{ + "operation": { + "type": "queryAwsApi", + "instanceType": "m6g.xlarge", + "region": "us-east-1" + } +} +``` + +### 6. QueryCostExplorer - Account-specific pricing (with EDP/PPA!) + +```json +{ + "operation": { + "type": "queryCostExplorer", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "awsAccountId": "123456789012", + "roleArn": "arn:aws:iam::123456789012:role/pathfinder-pricing-access" + } +} +``` + +**This automatically includes EDP/PPA discounts** by querying actual costs from the customer's account! + +## Dual Pricing Model + +### Retail Pricing (Public AWS List Prices) + +**Cache Key:** `PK=INSTANCE#m6g.xlarge, SK=REGION#us-east-1#RETAIL` + +**Source:** AWS Pricing API + +**Use when:** +- No customer AWS account access +- Planning/estimates for new customers +- Baseline pricing comparisons + +**TTL:** 30 days + +### Account-Specific Pricing (Includes EDP/PPA Automatically!) + +**Cache Key:** `PK=INSTANCE#m6g.xlarge, SK=REGION#us-east-1#ACCOUNT#123456789012` + +**Source:** AWS Cost Explorer (from customer account) + +**Use when:** +- Customer grants IAM role access +- Need actual cost projections +- Want automatic EDP/PPA discount detection + +**TTL:** 7 days (more current) + +**How EDP/PPA Auto-Detection Works:** +1. Assume IAM role into customer's AWS account (requires trust policy) +2. Query Cost Explorer for past 30 days of actual usage +3. Calculate average hourly rate from real costs +4. This rate automatically includes any EDP/PPA discounts! +5. No manual discount configuration needed + +## DynamoDB Schema + +### Pricing Items + +```typescript +{ + // Keys + "PK": "INSTANCE#m6g.xlarge", + "SK": "REGION#us-east-1#RETAIL", // or "REGION#us-east-1#ACCOUNT#123456789012" + + // GSI for access queries + "GSI1PK": "PRICING", + "accessCount": 127, // Incremented on each query + + // Pricing data (stored as JSON) + "pricingData": "{...}", // Full PricingData struct serialized + + // Metadata (for querying) + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "lastUpdated": "2025-11-27T00:00:00Z", + "lastAccessed": "2025-11-27T10:30:00Z", + "firstCached": "2025-11-01T08:00:00Z", + + // TTL + "expiresAt": 1740614400 // 30 days for retail, 7 days for account-specific +} +``` + +### GSI: AccessCountIndex + +**Purpose:** Query most-accessed instances for refresh + +**Keys:** `GSI1PK=PRICING, accessCount (range key, numeric)` + +**Usage:** +```rust +// Get top 50 most-accessed instances +query() + .index_name("AccessCountIndex") + .key_condition_expression("GSI1PK = :pk") + .scan_index_forward(false) // Descending + .limit(50) +``` + +## Self-Learning Refresh Strategy + +**How it works:** + +1. **Enrichment drives caching** + - Server created โ†’ enrichment queries pricing + - Cache miss โ†’ fetch from AWS API โ†’ cache for 30 days + - Cache hit โ†’ fast response + +2. **Access counting tracks popularity** + - Every query increments `accessCount` + - Popular instances accumulate high counts + - Unpopular instances stay at low counts + +3. **14-day refresh uses actual patterns** + - Query top 50 by `accessCount` + - Refresh only what's actually being used + - Adapts automatically as usage changes + +**Result:** Cache reflects real usage patterns, no hardcoded lists! + +## IAM Requirements + +### This Lambda Needs: + +**DynamoDB:** +```json +{ + "Effect": "Allow", + "Action": [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:Query" + ], + "Resource": [ + "arn:aws:dynamodb:REGION:ACCOUNT:table/pathfinder-ENV-pricing", + "arn:aws:dynamodb:REGION:ACCOUNT:table/pathfinder-ENV-pricing/index/AccessCountIndex" + ] +} +``` + +**AWS Pricing API:** +```json +{ + "Effect": "Allow", + "Action": [ + "pricing:GetProducts", + "pricing:DescribeServices" + ], + "Resource": "*" +} +``` + +**STS (for assuming roles):** +```json +{ + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam::*:role/pathfinder-pricing-access" +} +``` + +### Customer Account Needs: + +**IAM Role:** `pathfinder-pricing-access` + +**Permissions:** +```json +{ + "Effect": "Allow", + "Action": [ + "ce:GetCostAndUsage", + "ce:GetCostForecast" + ], + "Resource": "*" +} +``` + +**Trust Policy:** +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::OUR-ACCOUNT:role/crud-pricing-lambda-role" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "pathfinder-unique-external-id" + } + } + }] +} +``` + +## Development + +### Build + +```bash +./build.sh + +# Or manually: +cargo lambda build --release --arm64 --output-format zip +``` + +### Test + +```bash +# Run unit tests +cargo test + +# Test locally +cargo run -- '{ + "operation": { + "type": "get", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "fetchIfMissing": true + } +}' +``` + +### Deploy + +```bash +cd terraform +terraform init +terraform apply \ + -var="environment=dev" \ + -var="aws_region=us-east-1" +``` + +## Testing Deployed Lambda + +### Get Retail Pricing (Cache Miss โ†’ Fetch from API) + +```bash +aws lambda invoke \ + --function-name airun-pathfinder-crud-pricing-dev \ + --payload '{ + "operation": { + "type": "get", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "retail", + "fetchIfMissing": true + } + }' \ + output.json + +cat output.json | jq . +``` + +### Get Account-Specific Pricing (With EDP/PPA) + +```bash +aws lambda invoke \ + --function-name airun-pathfinder-crud-pricing-dev \ + --payload '{ + "operation": { + "type": "queryCostExplorer", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "awsAccountId": "123456789012", + "roleArn": "arn:aws:iam::123456789012:role/pathfinder-pricing-access" + } + }' \ + output.json + +cat output.json | jq '.body.pricing.ec2Pricing.onDemand' +# Shows actual pricing with EDP/PPA discount included! +``` + +### List Most-Accessed Instances + +```bash +aws lambda invoke \ + --function-name airun-pathfinder-crud-pricing-dev \ + --payload '{ + "operation": { + "type": "listCommon", + "limit": 50, + "minAccessCount": 5 + } + }' \ + output.json + +cat output.json | jq '.body.instances | .[0:10]' +# Shows top 10 most-accessed instances +``` + +## Environment Variables + +```hcl +environment_variables = { + RUST_LOG = "info" + TABLE_NAME = "pathfinder-dev-pricing" +} +``` + +## Integration Examples + +### From tool-pricing-query + +```rust +// tool-pricing-query calls crud-pricing instead of direct DynamoDB + +let payload = json!({ + "operation": { + "type": "get", + "instanceType": instance_type, + "region": region, + "pricingType": "retail", + "fetchIfMissing": true + } +}); + +let result = lambda_client + .invoke() + .function_name("airun-pathfinder-crud-pricing-dev") + .payload(Blob::new(serde_json::to_vec(&payload)?)) + .send() + .await?; +``` + +### From enrichment-server-pricing + +```rust +// Enrichment lambda calls crud-pricing to get pricing + +let payload = json!({ + "operation": { + "type": "get", + "instanceType": "m6g.xlarge", + "region": "us-east-1", + "pricingType": "accountSpecific", + "awsAccountId": project.aws_account_id, + "fetchIfMissing": true + } +}); + +let pricing = invoke_crud_pricing(payload).await?; +``` + +### From tool-pricing-refresh + +```rust +// Refresh lambda discovers common instances + +// Step 1: Get most-accessed +let common = invoke_crud_pricing(json!({ + "operation": { + "type": "listCommon", + "limit": 50, + "minAccessCount": 5 + } +})).await?; + +// Step 2: Refresh each one +for instance in common.instances { + invoke_crud_pricing(json!({ + "operation": { + "type": "get", + "instanceType": instance.instanceType, + "region": instance.region, + "pricingType": "retail", + "fetchIfMissing": true // Forces refresh + } + })).await?; +} +``` + +## Cache Strategy + +### Retail Pricing +- **TTL:** 30 days +- **Refresh:** 14-day automatic (top 50 instances) +- **Coverage:** All instance types (on-demand fetch) + +### Account-Specific Pricing +- **TTL:** 7 days (shorter for more current costs) +- **Refresh:** Not automatic (requires account access) +- **Coverage:** Only instances with usage in customer account + +### Access Counting +- Incremented on every Get operation +- Powers self-learning refresh +- GSI allows efficient queries + +## Pricing API Details + +### AWS Pricing API (Retail) + +**What we get:** +- OnDemand hourly/monthly rates +- Reserved pricing (Standard/Convertible, 1yr/3yr, all payment options) +- Instance specs (vCPUs, memory, architecture) + +**What we DON'T get:** +- Spot pricing (need EC2 Spot Price API) +- Customer-specific discounts (need Cost Explorer) + +**Rate Limits:** +- 1,000,000 requests/month free +- Then $0.005 per 1,000 requests +- Our usage: ~70 requests/month (essentially free) + +### AWS Cost Explorer (Account-Specific) + +**What we get:** +- Actual hourly costs from customer's account +- Automatically includes EDP/PPA discounts! +- Real usage-based pricing + +**What we DON'T get:** +- Instance specs (we fetch separately from Pricing API) +- Reserved/Spot breakdowns (just blended costs) + +**Requirements:** +- Must assume role into customer account +- Customer must have Cost Explorer enabled +- Needs 30 days of usage history + +## Error Handling + +| Error | Cause | Mitigation | +|-------|-------|------------| +| `Pricing not found and fetch_if_missing=false` | Cache miss, no fetch | Set `fetchIfMissing=true` | +| `AWS Pricing API call failed` | API throttling, network issue | Retry with exponential backoff | +| `Failed to assume role` | IAM permissions, invalid role ARN | Check trust policy and role exists | +| `No usage data found` | No historical usage in account | Fall back to retail pricing | +| `No pricing found for instance` | Invalid instance type | Validate instance type exists | + +## Performance + +**Expected latencies:** +- Cache hit: 10-20ms (DynamoDB query) +- Cache miss (Pricing API): 2-3 seconds (API call + parse) +- Cost Explorer: 3-5 seconds (assume role + API call) + +**Access count update:** Async (doesn't add latency) + +## Code Structure + +``` +src/ +โ”œโ”€โ”€ main.rs # Lambda handler and operation router +โ”œโ”€โ”€ models.rs # Data types and schemas +โ”œโ”€โ”€ db.rs # DynamoDB CRUD operations +โ”œโ”€โ”€ aws_pricing.rs # AWS Pricing API client +โ””โ”€โ”€ cost_explorer.rs # Cost Explorer client (EDP/PPA detection) +``` + +**Total:** ~1,100 lines of well-structured Rust + +## Testing Strategy + +**Unit Tests:** +- [ ] Test all DynamoDB operations +- [ ] Test pricing API response parsing +- [ ] Test Cost Explorer response parsing +- [ ] Test key generation (retail vs account-specific) +- [ ] Test expiration logic +- [ ] Test access counting + +**Integration Tests:** +- [ ] Test full Get operation (cache miss โ†’ fetch โ†’ cache) +- [ ] Test access count increment +- [ ] Test ListCommon with various filters +- [ ] Test account-specific pricing flow + +## Monitoring + +**CloudWatch Metrics to Add:** + +```rust +// Custom metrics +putMetric("CrudPricing", { + "CacheHitRate": hits / total * 100, + "AvgResponseTime": avg_latency_ms, + "ApiCallCount": api_calls, + "AccessCountUpdates": count_updates, +}); +``` + +**Alarms:** +- High cache miss rate (>20%) +- Slow response time (>5s) +- API failures (>5 in 5 minutes) + +## Related Components + +- `tool-pricing-query` - MCP tool that wraps this CRUD service +- `enrichment-server-pricing` - Uses this to get pricing during enrichment +- `tool-pricing-refresh` - Uses ListCommon to discover instances to refresh +- `pathfinder-dev-pricing` - DynamoDB table managed by this service + +## Next Steps + +After deployment: +1. Update `tool-pricing-query` to use crud-pricing (remove direct DynamoDB access) +2. Create `enrichment-server-pricing` that uses crud-pricing +3. Create `tool-pricing-refresh` that uses ListCommon operation +4. Update DynamoDB table with AccessCountIndex GSI + +See `PRICING-ENHANCEMENTS-PLAN.md` for complete roadmap. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..2b149f2 --- /dev/null +++ b/build.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +echo "๐Ÿ”จ Building crud-pricing for ARM64 (Graviton)..." + +# Check if cargo-lambda is installed +if ! command -v cargo-lambda &> /dev/null; then + echo "โŒ 'cargo-lambda' is not installed. Installing..." + cargo install cargo-lambda +fi + +# Check if cargo-audit is installed +if ! command -v cargo-audit &> /dev/null; then + echo "โŒ 'cargo-audit' is not installed. Installing..." + cargo install cargo-audit +fi + +# Run security audit before building +echo "๐Ÿ”’ Running security audit..." +cargo audit + +# Build for Lambda (ARM64) +echo "๐Ÿ“ฆ Compiling with cargo lambda..." +cargo lambda build --release --arm64 --output-format zip + +echo "โœ… Build complete: target/lambda/bootstrap/bootstrap.zip" +echo "" +echo "๐Ÿ“Š Package size:" +ls -lh target/lambda/bootstrap/bootstrap.zip +echo "" +echo "Next steps:" +echo " cd terraform" +echo " terraform init" +echo " terraform apply" diff --git a/src/aws_pricing.rs b/src/aws_pricing.rs new file mode 100644 index 0000000..1483fe3 --- /dev/null +++ b/src/aws_pricing.rs @@ -0,0 +1,383 @@ +use aws_sdk_pricing::types::Filter; +use aws_sdk_pricing::Client as PricingClient; +use chrono::Utc; +use serde_json::Value; +use std::collections::HashMap; +use tracing::{error, info, warn}; + +use crate::models::{ + Ec2Pricing, OnDemandPricing, PricingData, PricingType, ReservedOption, ReservedPricing, + ReservedTerm, +}; + +/// Fetch EC2 instance pricing from AWS Pricing API +pub async fn fetch_ec2_pricing( + client: &PricingClient, + instance_type: &str, + region: &str, +) -> Result { + info!("Fetching pricing from AWS Pricing API for {} in {}", instance_type, region); + + let location_name = region_to_location_name(region); + + let response = client + .get_products() + .service_code("AmazonEC2") + .filters( + Filter::builder() + .field("instanceType") + .value(instance_type) + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + Filter::builder() + .field("location") + .value(location_name) + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + Filter::builder() + .field("operatingSystem") + .value("Linux") + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + Filter::builder() + .field("tenancy") + .value("Shared") + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + Filter::builder() + .field("capacitystatus") + .value("Used") + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + Filter::builder() + .field("preInstalledSw") + .value("NA") + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .send() + .await + .map_err(|e| format!("AWS Pricing API call failed: {}", e))?; + + let price_list = response.price_list(); + + if price_list.is_empty() { + return Err(format!("No pricing found for {} in {}", instance_type, region)); + } + + info!("Parsing {} pricing items", price_list.len()); + + // Parse the first (and usually only) price item + let pricing = parse_pricing_response(price_list[0].as_str(), instance_type, region)?; + + Ok(pricing) +} + +/// Parse AWS Pricing API response +fn parse_pricing_response( + price_json: &str, + instance_type: &str, + region: &str, +) -> Result { + let item: Value = serde_json::from_str(price_json) + .map_err(|e| format!("Failed to parse pricing JSON: {}", e))?; + + // Extract instance attributes + let attributes = item["product"]["attributes"] + .as_object() + .ok_or("Missing product attributes")?; + + let vcpus: i32 = attributes + .get("vcpu") + .and_then(|v| v.as_str()) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let memory_str = attributes + .get("memory") + .and_then(|v| v.as_str()) + .unwrap_or("0 GiB"); + let memory_gb: f64 = memory_str + .replace(" GiB", "") + .replace(",", "") + .parse() + .unwrap_or(0.0); + + let instance_family = attributes + .get("instanceFamily") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let processor_architecture = attributes + .get("processorArchitecture") + .and_then(|v| v.as_str()) + .unwrap_or("x86_64"); + + let architectures = vec![processor_architecture.to_string()]; + + // Parse OnDemand pricing + let on_demand = parse_on_demand_pricing(&item)?; + + // Parse Reserved pricing (optional) + let reserved = parse_reserved_pricing(&item).ok(); + + // Spot pricing not available in Pricing API (would need EC2 Spot Price API) + let spot = None; + + let ec2_pricing = Ec2Pricing { + instance_family, + vcpus, + memory_gb, + architectures, + on_demand, + reserved, + spot, + }; + + let pricing_data = PricingData { + instance_type: instance_type.to_string(), + region: region.to_string(), + pricing_type: PricingType::Retail, + aws_account_id: None, + ec2_pricing, + source: "aws-pricing-api".to_string(), + last_updated: Utc::now().to_rfc3339(), + access_count: 0, + last_accessed: None, + first_cached: Some(Utc::now().to_rfc3339()), + }; + + Ok(pricing_data) +} + +/// Parse OnDemand pricing from terms +fn parse_on_demand_pricing(item: &Value) -> Result { + let on_demand_terms = item["terms"]["OnDemand"] + .as_object() + .ok_or("Missing OnDemand terms")?; + + // Get first (and usually only) term + let term = on_demand_terms + .values() + .next() + .ok_or("No OnDemand term found")?; + + let price_dimensions = term["priceDimensions"] + .as_object() + .ok_or("Missing priceDimensions")?; + + let price_dim = price_dimensions + .values() + .next() + .ok_or("No price dimension found")?; + + let hourly_str = price_dim["pricePerUnit"]["USD"] + .as_str() + .ok_or("Missing USD price")?; + + let hourly: f64 = hourly_str + .parse() + .map_err(|e| format!("Failed to parse hourly price: {}", e))?; + + let monthly = hourly * 730.0; // Hours per month + + Ok(OnDemandPricing { hourly, monthly }) +} + +/// Parse Reserved pricing from terms +fn parse_reserved_pricing(item: &Value) -> Result { + let reserved_terms = item["terms"]["Reserved"] + .as_object() + .ok_or("Missing Reserved terms")?; + + let mut standard: HashMap = HashMap::new(); + let mut convertible: HashMap = HashMap::new(); + + for (_, term) in reserved_terms { + let term_attributes = term["termAttributes"] + .as_object() + .ok_or("Missing termAttributes")?; + + let lease_contract_length = term_attributes + .get("LeaseContractLength") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let purchase_option = term_attributes + .get("PurchaseOption") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let offering_class = term_attributes + .get("OfferingClass") + .and_then(|v| v.as_str()) + .unwrap_or("standard"); + + // Parse the pricing + if let Ok(pricing) = parse_reserved_option(term) { + let term_key = if lease_contract_length.contains("1yr") || lease_contract_length.contains("1 year") { + "1yr" + } else if lease_contract_length.contains("3yr") || lease_contract_length.contains("3 year") { + "3yr" + } else { + continue; // Skip unknown terms + }; + + let option_type = if purchase_option.contains("All Upfront") { + "all_upfront" + } else if purchase_option.contains("Partial Upfront") { + "partial_upfront" + } else if purchase_option.contains("No Upfront") { + "no_upfront" + } else { + continue; // Skip unknown options + }; + + // Add to appropriate collection + let collection = if offering_class.contains("convertible") { + &mut convertible + } else { + &mut standard + }; + + let term_entry = collection + .entry(term_key.to_string()) + .or_insert_with(|| ReservedTerm { + all_upfront: ReservedOption::default(), + partial_upfront: ReservedOption::default(), + no_upfront: ReservedOption::default(), + }); + + match option_type { + "all_upfront" => term_entry.all_upfront = pricing, + "partial_upfront" => term_entry.partial_upfront = pricing, + "no_upfront" => term_entry.no_upfront = pricing, + _ => {} + } + } + } + + if standard.is_empty() && convertible.is_empty() { + return Err("No valid reserved pricing found".to_string()); + } + + Ok(ReservedPricing { + standard, + convertible, + }) +} + +/// Parse a single reserved pricing option +fn parse_reserved_option(term: &Value) -> Result { + let price_dimensions = term["priceDimensions"] + .as_object() + .ok_or("Missing priceDimensions")?; + + let mut total_upfront = 0.0; + let mut hourly_rate = 0.0; + + for (_, price_dim) in price_dimensions { + let unit = price_dim["unit"] + .as_str() + .unwrap_or(""); + + let price_str = price_dim["pricePerUnit"]["USD"] + .as_str() + .unwrap_or("0"); + + let price: f64 = price_str.parse().unwrap_or(0.0); + + if unit == "Quantity" { + total_upfront = price; + } else if unit == "Hrs" { + hourly_rate = price; + } + } + + let effective_hourly = if total_upfront > 0.0 { + // Calculate effective hourly from upfront + let term_attributes = term["termAttributes"] + .as_object() + .ok_or("Missing termAttributes")?; + + let lease_length = term_attributes + .get("LeaseContractLength") + .and_then(|v| v.as_str()) + .unwrap_or(""); + + let hours = if lease_length.contains("1yr") || lease_length.contains("1 year") { + 8760.0 // Hours in 1 year + } else if lease_length.contains("3yr") || lease_length.contains("3 year") { + 26280.0 // Hours in 3 years + } else { + 8760.0 // Default to 1 year + }; + + (total_upfront / hours) + hourly_rate + } else { + hourly_rate + }; + + let effective_monthly = effective_hourly * 730.0; + let monthly_payment = hourly_rate * 730.0; + + Ok(ReservedOption { + effective_hourly, + effective_monthly, + total_upfront, + monthly_payment, + }) +} + +impl Default for ReservedOption { + fn default() -> Self { + Self { + effective_hourly: 0.0, + effective_monthly: 0.0, + total_upfront: 0.0, + monthly_payment: 0.0, + } + } +} + +/// Convert AWS region code to location name used by Pricing API +pub fn region_to_location_name(region: &str) -> &str { + match region { + "us-east-1" => "US East (N. Virginia)", + "us-east-2" => "US East (Ohio)", + "us-west-1" => "US West (N. California)", + "us-west-2" => "US West (Oregon)", + "eu-west-1" => "EU (Ireland)", + "eu-west-2" => "EU (London)", + "eu-west-3" => "EU (Paris)", + "eu-central-1" => "EU (Frankfurt)", + "ap-southeast-1" => "Asia Pacific (Singapore)", + "ap-southeast-2" => "Asia Pacific (Sydney)", + "ap-northeast-1" => "Asia Pacific (Tokyo)", + "ap-northeast-2" => "Asia Pacific (Seoul)", + "ap-south-1" => "Asia Pacific (Mumbai)", + "sa-east-1" => "South America (Sao Paulo)", + "ca-central-1" => "Canada (Central)", + _ => { + warn!("Unknown region: {}, using as-is", region); + region + } + } +} diff --git a/src/cost_explorer.rs b/src/cost_explorer.rs new file mode 100644 index 0000000..8ae656b --- /dev/null +++ b/src/cost_explorer.rs @@ -0,0 +1,265 @@ +use aws_config::Region; +use aws_sdk_costexplorer::types::{ + DateInterval, Dimension, DimensionValues, Expression, Granularity, Metric, +}; +use aws_sdk_costexplorer::Client as CostExplorerClient; +use aws_sdk_sts::Client as StsClient; +use chrono::{Duration, Utc}; +use tracing::{error, info, warn}; + +use crate::models::{Ec2Pricing, OnDemandPricing, PricingData, PricingType}; + +/// Fetch account-specific pricing from Cost Explorer +/// This automatically includes EDP/PPA discounts! +pub async fn fetch_account_specific_pricing( + sts_client: &StsClient, + instance_type: &str, + region: &str, + aws_account_id: &str, + role_arn: &str, +) -> Result { + info!( + "Fetching account-specific pricing for {} in {} (account: {})", + instance_type, region, aws_account_id + ); + + // 1. Assume role into customer account + let credentials = sts_client + .assume_role() + .role_arn(role_arn) + .role_session_name("pathfinder-pricing-query") + .duration_seconds(900) // 15 minutes + .send() + .await + .map_err(|e| format!("Failed to assume role: {}", e))?; + + let creds = credentials + .credentials() + .ok_or("No credentials returned from assume role")?; + + info!("โœ… Successfully assumed role into account {}", aws_account_id); + + // 2. Create Cost Explorer client with assumed credentials + let config = aws_config::from_env() + .credentials_provider(aws_sdk_sts::config::Credentials::new( + creds.access_key_id(), + creds.secret_access_key(), + Some(creds.session_token().to_string()), + None, + "assumed-role", + )) + .region(Region::new(region.to_string())) + .load() + .await; + + let ce_client = CostExplorerClient::new(&config); + + // 3. Query Cost Explorer for historical usage and costs + let end_date = Utc::now(); + let start_date = end_date - Duration::days(30); + + info!( + "Querying Cost Explorer from {} to {}", + start_date.format("%Y-%m-%d"), + end_date.format("%Y-%m-%d") + ); + + let response = ce_client + .get_cost_and_usage() + .time_period( + DateInterval::builder() + .start(start_date.format("%Y-%m-%d").to_string()) + .end(end_date.format("%Y-%m-%d").to_string()) + .build() + .map_err(|e| format!("Failed to build date interval: {}", e))?, + ) + .granularity(Granularity::Daily) + .metrics(Metric::UnblendedCost) + .metrics(Metric::UsageQuantity) + .filter( + Expression::builder() + .dimensions( + DimensionValues::builder() + .key(Dimension::InstanceType) + .values(instance_type) + .build() + .map_err(|e| format!("Failed to build dimension: {}", e))?, + ) + .build(), + ) + .send() + .await + .map_err(|e| format!("Cost Explorer API call failed: {}", e))?; + + // 4. Calculate average hourly cost from actual usage + let results = response.results_by_time(); + + if results.is_empty() { + return Err(format!( + "No usage data found for {} in account {}. Instance may not be in use.", + instance_type, aws_account_id + )); + } + + let (total_cost, total_hours) = results.iter().fold((0.0, 0.0), |(cost_acc, hours_acc), result| { + let cost: f64 = result + .total() + .and_then(|t| t.get("UnblendedCost")) + .and_then(|m| m.amount()) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.0); + + let usage_quantity: f64 = result + .total() + .and_then(|t| t.get("UsageQuantity")) + .and_then(|m| m.amount()) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.0); + + (cost_acc + cost, hours_acc + usage_quantity) + }); + + if total_hours == 0.0 { + return Err(format!( + "No usage hours found for {} (zero usage in past 30 days)", + instance_type + )); + } + + let avg_hourly_cost = total_cost / total_hours; + let monthly_cost = avg_hourly_cost * 730.0; + + info!( + "โœ… Calculated account-specific pricing: ${:.4}/hr, ${:.2}/mo (from {} hours usage)", + avg_hourly_cost, monthly_cost, total_hours + ); + + // 5. Build pricing data + // Note: We don't have vcpu/memory from Cost Explorer, so we'd need to get that from Pricing API + // For now, we'll query Pricing API for instance specs and use Cost Explorer for pricing + let instance_specs = fetch_instance_specs(instance_type, region).await?; + + let ec2_pricing = Ec2Pricing { + instance_family: instance_specs.instance_family, + vcpus: instance_specs.vcpus, + memory_gb: instance_specs.memory_gb, + architectures: instance_specs.architectures, + on_demand: OnDemandPricing { + hourly: avg_hourly_cost, + monthly: monthly_cost, + }, + reserved: None, // Reserved pricing from Cost Explorer is complex, skip for now + spot: None, // Spot pricing not in Cost Explorer + }; + + let pricing_data = PricingData { + instance_type: instance_type.to_string(), + region: region.to_string(), + pricing_type: PricingType::AccountSpecific, + aws_account_id: Some(aws_account_id.to_string()), + ec2_pricing, + source: "cost-explorer".to_string(), + last_updated: Utc::now().to_rfc3339(), + access_count: 0, + last_accessed: None, + first_cached: Some(Utc::now().to_rfc3339()), + }; + + Ok(pricing_data) +} + +/// Simple struct for instance specs +struct InstanceSpecs { + instance_family: String, + vcpus: i32, + memory_gb: f64, + architectures: Vec, +} + +/// Fetch instance specifications from Pricing API +/// We need this because Cost Explorer only gives us costs, not specs +async fn fetch_instance_specs( + instance_type: &str, + region: &str, +) -> Result { + info!("Fetching instance specs for {}", instance_type); + + // Create new pricing client from default config + let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let pricing_client = aws_sdk_pricing::Client::new(&config); + + let location_name = crate::aws_pricing::region_to_location_name(region); + + let response = pricing_client + .get_products() + .service_code("AmazonEC2") + .filters( + aws_sdk_pricing::types::Filter::builder() + .field("instanceType") + .value(instance_type) + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .filters( + aws_sdk_pricing::types::Filter::builder() + .field("location") + .value(location_name) + .r#type("TERM_MATCH") + .build() + .map_err(|e| format!("Failed to build filter: {}", e))?, + ) + .send() + .await + .map_err(|e| format!("Failed to fetch instance specs: {}", e))?; + + let price_list = response.price_list(); + + if price_list.is_empty() { + return Err(format!("No instance specs found for {}", instance_type)); + } + + // Parse just the attributes we need + let item: serde_json::Value = serde_json::from_str(price_list[0].as_str()) + .map_err(|e| format!("Failed to parse pricing JSON: {}", e))?; + + let attributes = item["product"]["attributes"] + .as_object() + .ok_or("Missing product attributes")?; + + let vcpus: i32 = attributes + .get("vcpu") + .and_then(|v| v.as_str()) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let memory_str = attributes + .get("memory") + .and_then(|v| v.as_str()) + .unwrap_or("0 GiB"); + let memory_gb: f64 = memory_str + .replace(" GiB", "") + .replace(",", "") + .parse() + .unwrap_or(0.0); + + let instance_family = attributes + .get("instanceFamily") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let processor_architecture = attributes + .get("processorArchitecture") + .and_then(|v| v.as_str()) + .unwrap_or("x86_64"); + + let architectures = vec![processor_architecture.to_string()]; + + Ok(InstanceSpecs { + instance_family, + vcpus, + memory_gb, + architectures, + }) +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..061bb15 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,272 @@ +use aws_sdk_dynamodb::types::AttributeValue; +use aws_sdk_dynamodb::Client as DynamoDbClient; +use chrono::Utc; +use std::collections::HashMap; +use tracing::{error, info}; + +use crate::models::{CommonInstance, PricingData, PricingType}; + +const TTL_DAYS_RETAIL: i64 = 30; +const TTL_DAYS_ACCOUNT_SPECIFIC: i64 = 7; + +/// Get pricing from DynamoDB cache +pub async fn get_pricing( + client: &DynamoDbClient, + table_name: &str, + instance_type: &str, + region: &str, + pricing_type: &PricingType, + aws_account_id: Option<&str>, +) -> Result, String> { + let (pk, sk) = build_keys(instance_type, region, pricing_type, aws_account_id); + + info!("Querying pricing: PK={}, SK={}", pk, sk); + + let result = client + .get_item() + .table_name(table_name) + .key("PK", AttributeValue::S(pk)) + .key("SK", AttributeValue::S(sk)) + .send() + .await + .map_err(|e| format!("Failed to query DynamoDB: {}", e))?; + + match result.item() { + Some(item) => { + let pricing = parse_pricing_item(item)?; + + // Check if expired + if is_expired(&pricing) { + info!("Pricing data is expired"); + Ok(None) + } else { + info!("Cache hit for {} in {}", instance_type, region); + Ok(Some(pricing)) + } + } + None => { + info!("Cache miss for {} in {}", instance_type, region); + Ok(None) + } + } +} + +/// Store pricing in DynamoDB cache +pub async fn put_pricing( + client: &DynamoDbClient, + table_name: &str, + pricing_data: &PricingData, +) -> Result<(), String> { + let (pk, sk) = build_keys( + &pricing_data.instance_type, + &pricing_data.region, + &pricing_data.pricing_type, + pricing_data.aws_account_id.as_deref(), + ); + + // Calculate expiration timestamp + let ttl_days = match pricing_data.pricing_type { + PricingType::Retail => TTL_DAYS_RETAIL, + PricingType::AccountSpecific => TTL_DAYS_ACCOUNT_SPECIFIC, + }; + + let expires_at = Utc::now().timestamp() + (ttl_days * 24 * 60 * 60); + + info!("Storing pricing: PK={}, SK={}, TTL={} days", pk, sk, ttl_days); + + let mut item = HashMap::new(); + item.insert("PK".to_string(), AttributeValue::S(pk)); + item.insert("SK".to_string(), AttributeValue::S(sk)); + item.insert("GSI1PK".to_string(), AttributeValue::S("PRICING".to_string())); + + // Store pricing data as JSON + let pricing_json = serde_json::to_string(pricing_data) + .map_err(|e| format!("Failed to serialize pricing data: {}", e))?; + item.insert("pricingData".to_string(), AttributeValue::S(pricing_json)); + + // Metadata fields + item.insert("instanceType".to_string(), AttributeValue::S(pricing_data.instance_type.clone())); + item.insert("region".to_string(), AttributeValue::S(pricing_data.region.clone())); + item.insert("pricingType".to_string(), AttributeValue::S(format!("{:?}", pricing_data.pricing_type))); + item.insert("lastUpdated".to_string(), AttributeValue::S(pricing_data.last_updated.clone())); + item.insert("expiresAt".to_string(), AttributeValue::N(expires_at.to_string())); + + // Access tracking + item.insert("accessCount".to_string(), AttributeValue::N(pricing_data.access_count.to_string())); + if let Some(last_accessed) = &pricing_data.last_accessed { + item.insert("lastAccessed".to_string(), AttributeValue::S(last_accessed.clone())); + } + if let Some(first_cached) = &pricing_data.first_cached { + item.insert("firstCached".to_string(), AttributeValue::S(first_cached.clone())); + } + + client + .put_item() + .table_name(table_name) + .set_item(Some(item)) + .send() + .await + .map_err(|e| format!("Failed to put pricing: {}", e))?; + + info!("โœ… Pricing cached successfully"); + Ok(()) +} + +/// Query most accessed instances (for refresh) +pub async fn query_most_accessed( + client: &DynamoDbClient, + table_name: &str, + limit: usize, + min_access_count: Option, +) -> Result, String> { + info!("Querying most accessed instances (limit={}, min_count={:?})", limit, min_access_count); + + let mut query = client + .query() + .table_name(table_name) + .index_name("AccessCountIndex") + .key_condition_expression("GSI1PK = :pk") + .expression_attribute_values(":pk", AttributeValue::S("PRICING".to_string())) + .scan_index_forward(false) // Descending order (highest access count first) + .limit(limit as i32); + + // Optional: filter by minimum access count + if let Some(min_count) = min_access_count { + query = query + .filter_expression("accessCount >= :min") + .expression_attribute_values(":min", AttributeValue::N(min_count.to_string())); + } + + let result = query + .send() + .await + .map_err(|e| format!("Failed to query most accessed: {}", e))?; + + let items = result.items().iter() + .filter_map(|item| parse_common_instance(item).ok()) + .collect(); + + Ok(items) +} + +/// Increment access count for an instance +pub async fn increment_access_count( + client: &DynamoDbClient, + table_name: &str, + instance_type: &str, + region: &str, + pricing_type: &PricingType, + aws_account_id: Option<&str>, +) -> Result<(), String> { + let (pk, sk) = build_keys(instance_type, region, pricing_type, aws_account_id); + let now = Utc::now().to_rfc3339(); + + // Increment access count and update last accessed + client + .update_item() + .table_name(table_name) + .key("PK", AttributeValue::S(pk)) + .key("SK", AttributeValue::S(sk)) + .update_expression("SET accessCount = if_not_exists(accessCount, :zero) + :inc, lastAccessed = :now") + .expression_attribute_values(":inc", AttributeValue::N("1".to_string())) + .expression_attribute_values(":zero", AttributeValue::N("0".to_string())) + .expression_attribute_values(":now", AttributeValue::S(now)) + .send() + .await + .map_err(|e| format!("Failed to increment access count: {}", e))?; + + Ok(()) +} + +/// Build DynamoDB keys +fn build_keys( + instance_type: &str, + region: &str, + pricing_type: &PricingType, + aws_account_id: Option<&str>, +) -> (String, String) { + let pk = format!("INSTANCE#{}", instance_type); + + let sk = match (pricing_type, aws_account_id) { + (PricingType::Retail, _) => { + format!("REGION#{}#RETAIL", region) + } + (PricingType::AccountSpecific, Some(account_id)) => { + format!("REGION#{}#ACCOUNT#{}", region, account_id) + } + (PricingType::AccountSpecific, None) => { + // Fallback to retail if account ID missing + format!("REGION#{}#RETAIL", region) + } + }; + + (pk, sk) +} + +/// Parse pricing data from DynamoDB item +fn parse_pricing_item(item: &HashMap) -> Result { + let pricing_json = item + .get("pricingData") + .and_then(|v| v.as_s().ok()) + .ok_or("Missing pricingData field")?; + + let pricing: PricingData = serde_json::from_str(pricing_json) + .map_err(|e| format!("Failed to parse pricing data: {}", e))?; + + Ok(pricing) +} + +/// Parse common instance from DynamoDB item +fn parse_common_instance(item: &HashMap) -> Result { + let instance_type = item + .get("instanceType") + .and_then(|v| v.as_s().ok()) + .ok_or("Missing instanceType")? + .to_string(); + + let region = item + .get("region") + .and_then(|v| v.as_s().ok()) + .ok_or("Missing region")? + .to_string(); + + let access_count = item + .get("accessCount") + .and_then(|v| v.as_n().ok()) + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + + let last_accessed = item + .get("lastAccessed") + .and_then(|v| v.as_s().ok()) + .map(|s| s.to_string()); + + let last_updated = item + .get("lastUpdated") + .and_then(|v| v.as_s().ok()) + .ok_or("Missing lastUpdated")? + .to_string(); + + Ok(CommonInstance { + instance_type, + region, + access_count, + last_accessed, + last_updated, + }) +} + +/// Check if pricing data is expired +fn is_expired(pricing: &PricingData) -> bool { + let ttl_days = match pricing.pricing_type { + PricingType::Retail => TTL_DAYS_RETAIL, + PricingType::AccountSpecific => TTL_DAYS_ACCOUNT_SPECIFIC, + }; + + let expires_at = chrono::DateTime::parse_from_rfc3339(&pricing.last_updated) + .map(|dt| dt.timestamp() + (ttl_days * 24 * 60 * 60)) + .unwrap_or(0); + + let now = Utc::now().timestamp(); + now > expires_at +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..4785c63 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,287 @@ +mod aws_pricing; +mod cost_explorer; +mod db; +mod models; + +use aws_config::BehaviorVersion; +use aws_sdk_dynamodb::Client as DynamoDbClient; +use aws_sdk_pricing::Client as PricingClient; +use aws_sdk_sts::Client as StsClient; +use lambda_runtime::{service_fn, Error, LambdaEvent}; +use serde_json::Value; +use std::env; +use tracing::{error, info}; + +use crate::models::{PricingOperation, PricingRequest, PricingResponse, PricingType}; + +#[tokio::main] +async fn main() -> Result<(), Error> { + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_target(false) + .without_time() + .json() + .init(); + + lambda_runtime::run(service_fn(function_handler)).await +} + +async fn function_handler(event: LambdaEvent) -> Result { + let (payload, _context) = event.into_parts(); + + info!("Received pricing request: {:?}", payload); + + // Parse request + let request: PricingRequest = match serde_json::from_value(payload) { + Ok(req) => req, + Err(e) => { + error!("Invalid request: {}", e); + let response = PricingResponse::error(400, &format!("Invalid request: {}", e)); + return Ok(serde_json::to_value(response)?); + } + }; + + // Load AWS config and create clients + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let dynamodb_client = DynamoDbClient::new(&config); + let pricing_client = PricingClient::new(&config); + let sts_client = StsClient::new(&config); + + let table_name = env::var("TABLE_NAME").unwrap_or_else(|_| "pathfinder-dev-pricing".to_string()); + + // Route operation + let result = handle_operation( + request.operation, + &dynamodb_client, + &pricing_client, + &sts_client, + &table_name, + ) + .await; + + let response = match result { + Ok(data) => PricingResponse::success(data), + Err(e) => { + error!("Operation failed: {}", e); + PricingResponse::error(500, &e) + } + }; + + Ok(serde_json::to_value(response)?) +} + +async fn handle_operation( + operation: PricingOperation, + dynamodb_client: &DynamoDbClient, + pricing_client: &PricingClient, + sts_client: &StsClient, + table_name: &str, +) -> Result { + match operation { + PricingOperation::Get { + instance_type, + region, + pricing_type, + aws_account_id, + fetch_if_missing, + } => { + info!("Operation: Get (type={:?}, fetch_if_missing={})", pricing_type, fetch_if_missing); + + // Check cache + let cached = db::get_pricing( + dynamodb_client, + table_name, + &instance_type, + ®ion, + &pricing_type, + aws_account_id.as_deref(), + ) + .await?; + + match cached { + Some(mut pricing) => { + // Cache hit - increment access count asynchronously + let client = dynamodb_client.clone(); + let table = table_name.to_string(); + let inst = instance_type.clone(); + let reg = region.clone(); + let pt = pricing_type.clone(); + let acc_id = aws_account_id.clone(); + + tokio::spawn(async move { + let _ = db::increment_access_count( + &client, + &table, + &inst, + ®, + &pt, + acc_id.as_deref(), + ) + .await; + }); + + pricing.access_count += 1; // Show incremented count in response + Ok(serde_json::json!({ + "pricing": pricing, + "cacheStatus": "hit" + })) + } + None if fetch_if_missing => { + // Cache miss - fetch from AWS + info!("Cache miss, fetching from AWS API"); + + let pricing = fetch_pricing( + pricing_client, + sts_client, + &instance_type, + ®ion, + &pricing_type, + aws_account_id.as_deref(), + ) + .await?; + + // Cache it + db::put_pricing(dynamodb_client, table_name, &pricing).await?; + + Ok(serde_json::json!({ + "pricing": pricing, + "cacheStatus": "miss" + })) + } + None => { + Err("Pricing not found and fetch_if_missing=false".to_string()) + } + } + } + + PricingOperation::Put { + instance_type, + region, + pricing_type, + pricing_data, + } => { + info!("Operation: Put ({} in {})", instance_type, region); + + db::put_pricing(dynamodb_client, table_name, &pricing_data).await?; + + Ok(serde_json::json!({ + "success": true, + "instanceType": instance_type, + "region": region, + "pricingType": pricing_type + })) + } + + PricingOperation::ListCommon { + limit, + min_access_count, + } => { + info!("Operation: ListCommon (limit={}, min_access={})", limit, min_access_count.unwrap_or(0)); + + let common = db::query_most_accessed(dynamodb_client, table_name, limit, min_access_count).await?; + + Ok(serde_json::json!({ + "instances": common, + "count": common.len() + })) + } + + PricingOperation::IncrementAccess { + instance_type, + region, + } => { + info!("Operation: IncrementAccess ({} in {})", instance_type, region); + + // Default to retail for access counting + db::increment_access_count( + dynamodb_client, + table_name, + &instance_type, + ®ion, + &PricingType::Retail, + None, + ) + .await?; + + Ok(serde_json::json!({ + "success": true, + "instanceType": instance_type, + "region": region + })) + } + + PricingOperation::QueryAwsApi { + instance_type, + region, + } => { + info!("Operation: QueryAwsApi ({} in {})", instance_type, region); + + let pricing = aws_pricing::fetch_ec2_pricing(pricing_client, &instance_type, ®ion).await?; + + Ok(serde_json::json!({ + "pricing": pricing, + "source": "aws-pricing-api" + })) + } + + PricingOperation::QueryCostExplorer { + instance_type, + region, + aws_account_id, + role_arn, + } => { + info!( + "Operation: QueryCostExplorer ({} in {}, account={})", + instance_type, region, aws_account_id + ); + + let pricing = cost_explorer::fetch_account_specific_pricing( + sts_client, + &instance_type, + ®ion, + &aws_account_id, + &role_arn, + ) + .await?; + + Ok(serde_json::json!({ + "pricing": pricing, + "source": "cost-explorer", + "includesEDP": true + })) + } + } +} + +/// Fetch pricing based on type (retail or account-specific) +async fn fetch_pricing( + pricing_client: &PricingClient, + sts_client: &StsClient, + instance_type: &str, + region: &str, + pricing_type: &PricingType, + aws_account_id: Option<&str>, +) -> Result { + match pricing_type { + PricingType::Retail => { + info!("Fetching retail pricing from AWS Pricing API"); + aws_pricing::fetch_ec2_pricing(pricing_client, instance_type, region).await + } + PricingType::AccountSpecific => { + let account_id = aws_account_id.ok_or("aws_account_id required for account-specific pricing")?; + + // Role ARN is expected to be in format: arn:aws:iam::{account_id}:role/pathfinder-pricing-access + let role_arn = format!("arn:aws:iam::{}:role/pathfinder-pricing-access", account_id); + + info!("Fetching account-specific pricing from Cost Explorer"); + cost_explorer::fetch_account_specific_pricing( + sts_client, + instance_type, + region, + account_id, + &role_arn, + ) + .await + } + } +} diff --git a/src/models.rs b/src/models.rs new file mode 100644 index 0000000..89d0db4 --- /dev/null +++ b/src/models.rs @@ -0,0 +1,199 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Request payload for the Lambda function +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PricingRequest { + pub operation: PricingOperation, +} + +/// Operations supported by crud-pricing +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum PricingOperation { + /// Get pricing from cache (optionally fetch from AWS if missing) + Get { + instance_type: String, + region: String, + pricing_type: PricingType, + #[serde(default)] + aws_account_id: Option, + #[serde(default)] + fetch_if_missing: bool, + }, + + /// Put pricing data into cache + Put { + instance_type: String, + region: String, + pricing_type: PricingType, + pricing_data: PricingData, + }, + + /// List most commonly accessed instances + ListCommon { + #[serde(default = "default_limit")] + limit: usize, + #[serde(default)] + min_access_count: Option, + }, + + /// Increment access count for an instance + IncrementAccess { + instance_type: String, + region: String, + }, + + /// Query AWS Pricing API directly + QueryAwsApi { + instance_type: String, + region: String, + }, + + /// Query AWS Cost Explorer for account-specific pricing + QueryCostExplorer { + instance_type: String, + region: String, + aws_account_id: String, + role_arn: String, + }, +} + +fn default_limit() -> usize { + 50 +} + +/// Type of pricing data +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum PricingType { + /// Public AWS list prices + Retail, + /// Account-specific prices (includes EDP/PPA automatically) + AccountSpecific, +} + +/// Complete pricing data for an instance +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PricingData { + pub instance_type: String, + pub region: String, + pub pricing_type: PricingType, + + #[serde(skip_serializing_if = "Option::is_none")] + pub aws_account_id: Option, + + /// EC2 instance pricing + pub ec2_pricing: Ec2Pricing, + + /// Metadata + pub source: String, // "aws-pricing-api" or "cost-explorer" + pub last_updated: String, + + /// Cache/access tracking + #[serde(default)] + pub access_count: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_accessed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_cached: Option, +} + +/// EC2 instance pricing details +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Ec2Pricing { + pub instance_family: String, // "m6g", "t3", etc. + pub vcpus: i32, + pub memory_gb: f64, + pub architectures: Vec, // ["x86_64"], ["arm64"], etc. + + /// OnDemand pricing + pub on_demand: OnDemandPricing, + + /// Reserved pricing (if available) + #[serde(skip_serializing_if = "Option::is_none")] + pub reserved: Option, + + /// Spot pricing (if available) + #[serde(skip_serializing_if = "Option::is_none")] + pub spot: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OnDemandPricing { + pub hourly: f64, + pub monthly: f64, // hourly * 730 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReservedPricing { + pub standard: HashMap, // "1yr", "3yr" + pub convertible: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReservedTerm { + pub all_upfront: ReservedOption, + pub partial_upfront: ReservedOption, + pub no_upfront: ReservedOption, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReservedOption { + pub effective_hourly: f64, + pub effective_monthly: f64, + pub total_upfront: f64, + pub monthly_payment: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpotPricing { + pub current_hourly: f64, + pub avg_hourly: f64, // 30-day average + pub max_hourly: f64, + pub interruption_frequency: String, // "<5%", "5-10%", etc. + pub savings_vs_on_demand_percent: i32, +} + +/// Common instance returned by ListCommon operation +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CommonInstance { + pub instance_type: String, + pub region: String, + pub access_count: u32, + pub last_accessed: Option, + pub last_updated: String, +} + +/// Response envelope +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PricingResponse { + pub status_code: u16, + pub body: serde_json::Value, +} + +impl PricingResponse { + pub fn success(data: impl Serialize) -> Self { + Self { + status_code: 200, + body: serde_json::to_value(data).unwrap_or(serde_json::Value::Null), + } + } + + pub fn error(status_code: u16, message: &str) -> Self { + Self { + status_code, + body: serde_json::json!({ "error": message }), + } + } +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..79ee204 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,171 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.23" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Project = "airun-pathfinder" + Component = "crud" + Service = "pricing" + Environment = var.environment + ManagedBy = "terraform" + } + } +} + +# Get current AWS account ID +data "aws_caller_identity" "current" {} + +# Lambda function +resource "aws_lambda_function" "crud_pricing" { + filename = "../target/lambda/bootstrap/bootstrap.zip" + function_name = "airun-pathfinder-crud-pricing-${var.environment}" + role = aws_iam_role.lambda.arn + handler = "bootstrap" + runtime = "provided.al2023" + architectures = ["arm64"] + timeout = var.lambda_timeout + memory_size = var.lambda_memory + + source_code_hash = filebase64sha256("../target/lambda/bootstrap/bootstrap.zip") + + environment { + variables = { + RUST_LOG = var.log_level + TABLE_NAME = local.table_name + ENVIRONMENT = var.environment + } + } + + # Enable X-Ray tracing for observability + tracing_config { + mode = "Active" + } + + tags = { + Name = "airun-pathfinder-crud-pricing" + } +} + +# CloudWatch Log Group +resource "aws_cloudwatch_log_group" "crud_pricing" { + name = "/aws/lambda/${aws_lambda_function.crud_pricing.function_name}" + retention_in_days = var.log_retention_days + + tags = { + Name = "crud-pricing-logs" + } +} + +# IAM Role for Lambda +resource "aws_iam_role" "lambda" { + name = "airun-pathfinder-crud-pricing-role-${var.environment}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + }] + }) + + tags = { + Name = "airun-pathfinder-crud-pricing-role" + } +} + +# Basic Lambda execution policy +resource "aws_iam_role_policy_attachment" "lambda_basic" { + role = aws_iam_role.lambda.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +# X-Ray permissions for tracing +resource "aws_iam_role_policy_attachment" "lambda_xray" { + role = aws_iam_role.lambda.name + policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess" +} + +# DynamoDB access policy +resource "aws_iam_role_policy" "dynamodb_access" { + name = "airun-pathfinder-crud-pricing-dynamodb-policy" + role = aws_iam_role.lambda.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:Query" + ] + Resource = [ + "arn:aws:dynamodb:${var.aws_region}:${data.aws_caller_identity.current.account_id}:table/${local.table_name}", + "arn:aws:dynamodb:${var.aws_region}:${data.aws_caller_identity.current.account_id}:table/${local.table_name}/index/AccessCountIndex" + ] + } + ] + }) +} + +# AWS Pricing API access +resource "aws_iam_role_policy" "pricing_api_access" { + name = "airun-pathfinder-crud-pricing-pricing-api-policy" + role = aws_iam_role.lambda.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "pricing:GetProducts", + "pricing:DescribeServices", + "pricing:GetAttributeValues" + ] + Resource = "*" + } + ] + }) +} + +# STS AssumeRole access (for Cost Explorer in customer accounts) +resource "aws_iam_role_policy" "sts_assume_role" { + name = "airun-pathfinder-crud-pricing-sts-policy" + role = aws_iam_role.lambda.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Resource = "arn:aws:iam::*:role/pathfinder-pricing-access" + } + ] + }) +} + +# Cost Explorer access (when assuming role) +# Note: This is granted via the role in the customer account, not here + +# Local variables +locals { + table_name = var.table_name != "" ? var.table_name : "pathfinder-${var.environment}-pricing" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..879ca8e --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "lambda_function_arn" { + description = "ARN of the crud-pricing Lambda function" + value = aws_lambda_function.crud_pricing.arn +} + +output "lambda_function_name" { + description = "Name of the crud-pricing Lambda function" + value = aws_lambda_function.crud_pricing.function_name +} + +output "lambda_role_arn" { + description = "ARN of the Lambda execution role" + value = aws_iam_role.lambda.arn +} + +output "table_name" { + description = "DynamoDB table name used by this Lambda" + value = local.table_name +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..a4c77cc --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,41 @@ +variable "environment" { + description = "Environment name (dev, staging, prod)" + type = string + default = "dev" +} + +variable "aws_region" { + description = "AWS region for deployment" + type = string + default = "us-east-1" +} + +variable "lambda_timeout" { + description = "Lambda timeout in seconds" + type = number + default = 60 +} + +variable "lambda_memory" { + description = "Lambda memory in MB" + type = number + default = 512 +} + +variable "log_level" { + description = "Rust log level" + type = string + default = "info" +} + +variable "log_retention_days" { + description = "CloudWatch log retention in days" + type = number + default = 7 +} + +variable "table_name" { + description = "DynamoDB table name for pricing" + type = string + default = "" +}