Restructure TravelAI module - numbered scripts with CREATE EXTERNAL MODEL

Author: SudhirAmin

labs/travelapp/03_deploy_embedding_proxy.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""
+Deploy the Bedrock Embedding Proxy (Lambda + API Gateway).
+This creates the serverless translation layer that allows SQL Server
+to call Bedrock via CREATE EXTERNAL MODEL + AI_GENERATE_EMBEDDINGS.
+
+Usage:
+    python3 03_deploy_embedding_proxy.py
+"""
+import boto3
+import json
+import os
+import time
+import zipfile
+import tempfile
+
+region = os.environ.get('AWS_REGION', 'us-west-2')
+account_id = boto3.client('sts').get_caller_identity()['Account']
+
+print("Deploying Bedrock Embedding Proxy...")
+print(f"  Region: {region}")
+print(f"  Account: {account_id}")
+
+iam = boto3.client('iam')
+lam = boto3.client('lambda', region_name=region)
+apigw = boto3.client('apigateway', region_name=region)
+
+# Step 1: Create IAM Role
+ROLE_NAME = 'bedrock-embedding-lambda-role'
+print("\n1. Creating IAM role...")
+try:
+    role = iam.create_role(
+        RoleName=ROLE_NAME,
+        AssumeRolePolicyDocument=json.dumps({
+            "Version": "2012-10-17",
+            "Statement": [{"Effect": "Allow", "Principal": {"Service": "lambda.amazonaws.com"}, "Action": "sts:AssumeRole"}]
+        })
+    )
+    iam.put_role_policy(RoleName=ROLE_NAME, PolicyName='BedrockAccess', PolicyDocument=json.dumps({
+        "Version": "2012-10-17",
+        "Statement": [
+            {"Effect": "Allow", "Action": ["bedrock:InvokeModel"], "Resource": "*"},
+            {"Effect": "Allow", "Action": ["logs:*"], "Resource": "*"}
+        ]
+    }))
+    print(f"   Role created: {role['Role']['Arn']}")
+    time.sleep(10)  # Wait for role propagation
+except iam.exceptions.EntityAlreadyExistsException:
+    print(f"   Role already exists")
+    role = iam.get_role(RoleName=ROLE_NAME)
+
+role_arn = f"arn:aws:iam::{account_id}:role/{ROLE_NAME}"
+
+# Step 2: Create Lambda Function
+FUNCTION_NAME = 'bedrock-embedding-proxy'
+print("\n2. Creating Lambda function...")
+
+lambda_code = '''
+import json
+import boto3
+
+bedrock = boto3.client('bedrock-runtime', region_name='us-west-2')
+
+def lambda_handler(event, context):
+    try:
+        body = json.loads(event.get('body', '{}'))
+        input_text = body.get('input', body.get('inputText', ''))
+        if isinstance(input_text, list):
+            input_text = input_text[0]
+        dimensions = body.get('dimensions', 1024)
+        
+        response = bedrock.invoke_model(
+            modelId='amazon.titan-embed-text-v2:0',
+            contentType='application/json',
+            accept='application/json',
+            body=json.dumps({'inputText': input_text[:8000], 'dimensions': dimensions})
+        )
+        
+        result = json.loads(response['body'].read())
+        embedding = result['embedding']
+        
+        return {
+            'statusCode': 200,
+            'headers': {'Content-Type': 'application/json'},
+            'body': json.dumps({
+                'object': 'list',
+                'data': [{'object': 'embedding', 'embedding': embedding, 'index': 0}],
+                'model': 'amazon.titan-embed-text-v2',
+                'usage': {'prompt_tokens': len(input_text.split()), 'total_tokens': len(input_text.split())}
+            })
+        }
+    except Exception as e:
+        return {
+            'statusCode': 500,
+            'headers': {'Content-Type': 'application/json'},
+            'body': json.dumps({'error': {'message': str(e), 'type': 'server_error'}})
+        }
+'''
+
+# Create zip
+with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp:
+    zip_path = tmp.name
+    with zipfile.ZipFile(zip_path, 'w') as zf:
+        zf.writestr('lambda_function.py', lambda_code)
+
+with open(zip_path, 'rb') as f:
+    zip_bytes = f.read()
+
+try:
+    lam.create_function(
+        FunctionName=FUNCTION_NAME, Runtime='python3.12',
+        Role=role_arn, Handler='lambda_function.lambda_handler',
+        Code={'ZipFile': zip_bytes}, Timeout=30, MemorySize=256
+    )
+    print(f"   Function created: {FUNCTION_NAME}")
+except lam.exceptions.ResourceConflictException:
+    lam.update_function_code(FunctionName=FUNCTION_NAME, ZipFile=zip_bytes)
+    lam.update_function_configuration(FunctionName=FUNCTION_NAME, Handler='lambda_function.lambda_handler')
+    print(f"   Function updated: {FUNCTION_NAME}")
+
+time.sleep(5)
+
+# Step 3: Create API Gateway
+print("\n3. Creating API Gateway...")
+apis = apigw.get_rest_apis()['items']
+existing = [a for a in apis if a['name'] == 'bedrock-embedding-api']
+
+if existing:
+    api_id = existing[0]['id']
+    print(f"   API already exists: {api_id}")
+else:
+    api = apigw.create_rest_api(name='bedrock-embedding-api', endpointConfiguration={'types': ['REGIONAL']})
+    api_id = api['id']
+    print(f"   API created: {api_id}")
+
+# Get root resource
+resources = apigw.get_resources(restApiId=api_id)['items']
+root_id = [r for r in resources if r['path'] == '/'][0]['id']
+
+# Create /embed resource if not exists
+embed_resources = [r for r in resources if r.get('pathPart') == 'embed']
+if embed_resources:
+    resource_id = embed_resources[0]['id']
+else:
+    resource = apigw.create_resource(restApiId=api_id, parentId=root_id, pathPart='embed')
+    resource_id = resource['id']
+
+# Create POST method
+try:
+    apigw.put_method(restApiId=api_id, resourceId=resource_id, httpMethod='POST', authorizationType='NONE')
+except:
+    pass
+
+# Create Lambda integration
+lambda_uri = f"arn:aws:apigateway:{region}:lambda:path/2015-03-31/functions/arn:aws:lambda:{region}:{account_id}:function:{FUNCTION_NAME}/invocations"
+try:
+    apigw.put_integration(restApiId=api_id, resourceId=resource_id, httpMethod='POST', type='AWS_PROXY', integrationHttpMethod='POST', uri=lambda_uri)
+except:
+    pass
+
+# Add Lambda permission
+try:
+    lam.add_permission(FunctionName=FUNCTION_NAME, StatementId='apigateway-invoke', Action='lambda:InvokeFunction', Principal='apigateway.amazonaws.com')
+except:
+    pass
+
+# Deploy
+apigw.create_deployment(restApiId=api_id, stageName='prod')
+
+endpoint = f"https://{api_id}.execute-api.{region}.amazonaws.com/prod/embed"
+print(f"\n   Endpoint: {endpoint}")
+
+# Step 4: Save endpoint for SQL scripts
+env_file = '/tmp/embedding_endpoint.txt'
+with open(env_file, 'w') as f:
+    f.write(endpoint)
+
+print(f"\n{'='*60}")
+print(f"DONE! Embedding proxy deployed.")
+print(f"Endpoint: {endpoint}")
+print(f"\nNext: python3.11 run_sql_file.py load_generator/04_register_model.sql")
+print(f"{'='*60}")

…velapp/load_generator/travelai_setup.sql …pp/load_generator/01_create_database.sqllabs/travelapp/load_generator/travelai_setup.sql renamed to labs/travelapp/load_generator/01_create_database.sql labs/travelapp/load_generator/travelai_setup.sql renamed to labs/travelapp/load_generator/01_create_database.sql

@@ -1,6 +1,5 @@
--- TravelAI Setup: Database + Schema + Indexes + Seed Data
--- Run this first to create the TravelAI database with all tables and sample data.
--- Usage: python3.11 run_sql_file.py load_generator/travelai_setup.sql
+-- TravelAI Database Setup: CREATE DATABASE + Schema + Indexes + Seed Data
+-- Usage: python3.11 run_sql_file.py load_generator/01_create_database.sql
 
 IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = 'TravelAI')
     CREATE DATABASE TravelAI;

…pp/load_generator/travelai_app_procs.sql …enerator/02_create_search_procedures.sqllabs/travelapp/load_generator/travelai_app_procs.sql renamed to labs/travelapp/load_generator/02_create_search_procedures.sql labs/travelapp/load_generator/travelai_app_procs.sql renamed to labs/travelapp/load_generator/02_create_search_procedures.sql

@@ -1,17 +1,12 @@
--- TravelAI Search Stored Procedures
--- Created for the Search UI comparison demo
--- These run against the TravelAI database
-
+-- TravelAI Search Procedures
+-- Usage: python3.11 run_sql_file.py load_generator/02_create_search_procedures.sql
 USE TravelAI;
 GO
 
 SET NOCOUNT ON;
 GO
 
--- =============================================
 -- usp_SearchSQL: Pure WHERE clause matching
--- Maps keywords to climate categories
--- =============================================
 CREATE OR ALTER PROCEDURE dbo.usp_SearchSQL
     @QueryText NVARCHAR(1000),
     @TopK INT = 5
@@ -20,56 +15,41 @@ BEGIN
     SET NOCOUNT ON;
     DECLARE @Climate NVARCHAR(30) = NULL;
     IF @QueryText LIKE '%beach%' OR @QueryText LIKE '%tropical%' OR @QueryText LIKE '%island%'
-        SET @Climate = 'Tropical';
-    ELSE IF @QueryText LIKE '%mountain%' OR @QueryText LIKE '%hiking%'
-        SET @Climate = 'Temperate';
+        SET @Climate = 'tropical';
+    ELSE IF @QueryText LIKE '%mountain%' OR @QueryText LIKE '%hiking%' OR @QueryText LIKE '%alpine%'
+        SET @Climate = 'alpine';
     ELSE IF @QueryText LIKE '%desert%' OR @QueryText LIKE '%arid%'
-        SET @Climate = 'Arid';
-    ELSE IF @QueryText LIKE '%arctic%' OR @QueryText LIKE '%glacier%'
-        SET @Climate = 'Arctic';
+        SET @Climate = 'semi-arid';
 
     IF @Climate IS NOT NULL
         SELECT TOP (@TopK) destination_id, name AS Title, country_code AS Country, region AS Continent, climate AS Climate, best_season AS Season, LEFT(description,200) AS Snippet, popularity_score, 100 AS RelevanceScore
-        FROM Destinations WHERE Climate = @Climate ORDER BY popularity_score DESC;
+        FROM Destinations WHERE climate = @Climate ORDER BY popularity_score DESC;
     ELSE
         SELECT TOP (@TopK) destination_id, name AS Title, country_code AS Country, region AS Continent, climate AS Climate, best_season AS Season, LEFT(description,200) AS Snippet, popularity_score, 50 AS RelevanceScore
         FROM Destinations ORDER BY popularity_score DESC;
 END;
 GO
 
--- =============================================
--- usp_SearchLIKE: LIKE pattern matching
--- Splits first two words, searches description + name + Country
--- =============================================
+-- usp_SearchLIKE: Pattern matching
 CREATE OR ALTER PROCEDURE dbo.usp_SearchLIKE
     @QueryText NVARCHAR(1000),
     @TopK INT = 5
 AS
 BEGIN
     SET NOCOUNT ON;
     DECLARE @Word1 NVARCHAR(100) = LEFT(@QueryText, CHARINDEX(' ', @QueryText + ' ') - 1);
-    DECLARE @Word2 NVARCHAR(100) = NULL;
-    
-    IF CHARINDEX(' ', @QueryText) > 0
-        SET @Word2 = SUBSTRING(@QueryText, CHARINDEX(' ', @QueryText) + 1, 
-            CHARINDEX(' ', @QueryText + ' ', CHARINDEX(' ', @QueryText) + 1) - CHARINDEX(' ', @QueryText) - 1);
-
-    SELECT TOP (@TopK) 
-        destination_id, name AS Title, country_code AS Country, region AS Continent, climate AS Climate, best_season AS Season, 
-        description AS Snippet, popularity_score, 120 AS RelevanceScore
+    SELECT TOP (@TopK)
+        destination_id, name AS Title, country_code AS Country, region AS Continent, climate AS Climate, best_season AS Season,
+        LEFT(description,200) AS Snippet, popularity_score, 120 AS RelevanceScore
     FROM Destinations
-    WHERE Description LIKE '%' + @Word1 + '%'
-       OR (@Word2 IS NOT NULL AND Description LIKE '%' + @Word2 + '%')
+    WHERE description LIKE '%' + @Word1 + '%'
        OR name LIKE '%' + @Word1 + '%'
        OR country_code LIKE '%' + @Word1 + '%'
     ORDER BY popularity_score DESC;
 END;
 GO
 
--- =============================================
--- usp_SearchFreetext: Full-Text Search (FREETEXT only, no RAG)
--- Uses SQL Server Full-Text engine with stemming and word forms
--- =============================================
+-- usp_SearchFreetext: Full-Text Search
 CREATE OR ALTER PROCEDURE dbo.usp_SearchFreetext
     @QueryText NVARCHAR(1000),
     @TopK INT = 5
@@ -83,18 +63,13 @@ BEGIN
 END;
 GO
 
--- =============================================
 -- usp_TravelSearch: Hybrid (FREETEXT + RAG document chunks)
--- Returns 2 result sets: destinations + supporting document context
--- =============================================
 CREATE OR ALTER PROCEDURE dbo.usp_TravelSearch
     @QueryText NVARCHAR(1000),
     @TopK INT = 5
 AS
 BEGIN
     SET NOCOUNT ON;
-
-    -- Result set 1: Destinations ranked by Full-Text relevance
     SELECT TOP (@TopK)
         'Destination' AS ResultType,
         d.destination_id AS SourceID,
@@ -111,7 +86,6 @@ BEGIN
         ON d.destination_id = ft.[KEY]
     ORDER BY ft.[RANK] DESC;
 
-    -- Result set 2: RAG context from document chunks
     SELECT TOP 3
         'Document' AS ResultType,
         dc.chunk_id AS SourceID,
@@ -125,12 +99,7 @@ BEGIN
 END;
 GO
 
-PRINT 'Search SPs created: usp_SearchSQL, usp_SearchLIKE, usp_SearchFreetext, usp_TravelSearch';
-GO
-
--- =============================================
--- usp_SearchVector: Semantic vector search using VECTOR_DISTANCE
--- =============================================
+-- usp_SearchVector: Semantic vector search
 CREATE OR ALTER PROCEDURE dbo.usp_SearchVector
     @QueryEmbedding VECTOR(1024),
     @TopK INT = 5

labs/travelapp/load_generator/04_register_model.sql

@@ -0,0 +1,45 @@
+-- Register Bedrock Embedding Model via API Gateway proxy
+-- NOTE: Replace <API_GATEWAY_URL> with the endpoint from 03_deploy_embedding_proxy.py
+-- Usage: python3.11 run_sql_file.py load_generator/04_register_model.sql
+USE TravelAI;
+GO
+
+-- Create credential for API Gateway (no auth needed, just a placeholder header)
+IF EXISTS (SELECT 1 FROM sys.database_scoped_credentials WHERE name LIKE '%execute-api%')
+BEGIN
+    -- Drop model first if exists
+    IF EXISTS (SELECT 1 FROM sys.external_models WHERE name = 'bedrock_embed')
+        DROP EXTERNAL MODEL bedrock_embed;
+    
+    DECLARE @cred_name NVARCHAR(500);
+    SELECT @cred_name = name FROM sys.database_scoped_credentials WHERE name LIKE '%execute-api%';
+    EXEC('DROP DATABASE SCOPED CREDENTIAL [' + @cred_name + ']');
+END
+GO
+
+-- Read endpoint from file (written by 03_deploy_embedding_proxy.py)
+-- For manual setup, replace the URL below with your API Gateway endpoint
+DECLARE @endpoint NVARCHAR(500) = N'<API_GATEWAY_URL>';
+
+-- Create credential
+DECLARE @sql NVARCHAR(MAX) = N'
+CREATE DATABASE SCOPED CREDENTIAL [' + @endpoint + N']
+WITH IDENTITY = ''HTTPEndpointHeaders'',
+     SECRET = ''{"x-api-key":"none"}''';
+EXEC sp_executesql @sql;
+
+-- Create external model
+SET @sql = N'
+CREATE EXTERNAL MODEL bedrock_embed
+WITH (
+    LOCATION = ''' + @endpoint + N''',
+    API_FORMAT = ''OpenAI'',
+    MODEL_TYPE = EMBEDDINGS,
+    MODEL = ''amazon.titan-embed-text-v2'',
+    CREDENTIAL = [' + @endpoint + N']
+)';
+EXEC sp_executesql @sql;
+
+PRINT 'External model bedrock_embed registered successfully';
+PRINT 'Test: SELECT DATALENGTH(AI_GENERATE_EMBEDDINGS(N''hello'' USE MODEL bedrock_embed))';
+GO

labs/travelapp/load_generator/05_populate_vectors.sql

@@ -0,0 +1,22 @@
+-- Populate vector embeddings using AI_GENERATE_EMBEDDINGS
+-- Usage: python3.11 run_sql_file.py load_generator/05_populate_vectors.sql
+USE TravelAI;
+GO
+
+-- Embed all destinations (one line per table)
+UPDATE Destinations
+SET description_vector = AI_GENERATE_EMBEDDINGS(description USE MODEL bedrock_embed)
+WHERE description_vector IS NULL;
+GO
+
+-- Embed all document chunks
+UPDATE DocumentChunks
+SET content_vector = AI_GENERATE_EMBEDDINGS(content USE MODEL bedrock_embed)
+WHERE content_vector IS NULL;
+GO
+
+-- Verify
+SELECT 'Destinations' AS Source, COUNT(*) AS Embedded FROM Destinations WHERE description_vector IS NOT NULL
+UNION ALL
+SELECT 'DocumentChunks', COUNT(*) FROM DocumentChunks WHERE content_vector IS NOT NULL;
+GO