From 414e713aad944772f774371ac10ccd32a95f7dd7 Mon Sep 17 00:00:00 2001 From: Paul Yu Date: Tue, 23 Apr 2024 09:52:02 -0700 Subject: [PATCH 01/19] feat: adding image generation capabilities to ai-service --- src/ai-service/README.md | 6 ++- src/ai-service/main.py | 16 +++++-- src/ai-service/requirements.txt | 7 ++- src/ai-service/routers/image_generator.py | 58 +++++++++++++++++++++++ src/ai-service/test-ai-service.http | 10 ++++ 5 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 src/ai-service/routers/image_generator.py diff --git a/src/ai-service/README.md b/src/ai-service/README.md index 9765a24b..f5a878d3 100644 --- a/src/ai-service/README.md +++ b/src/ai-service/README.md @@ -23,10 +23,12 @@ source .venv/bin/activate pip install -r requirements.txt export USE_AZURE_OPENAI=True # set to False if you are not using Azure OpenAI -export USE_AZURE_AD=False # set to True if you are using Azure OpenAI with Azure AD authentication +export USE_AZURE_AD=True # set to True if you are using Azure OpenAI with Azure AD authentication +export AZURE_OPENAI_API_VERSION=2024-02-15-preview # set to the version of the Azure OpenAI API you are using https://learn.microsoft.com/azure/ai-services/openai/reference#rest-api-versioning +export AZURE_OPENAI_DALLE_DEPLOYMENT_NAME= # required if using Azure OpenAI export AZURE_OPENAI_DEPLOYMENT_NAME= # required if using Azure OpenAI export AZURE_OPENAI_ENDPOINT= # required if using Azure OpenAI -export OPENAI_API_KEY= # always required +export OPENAI_API_KEY= # always required if using OpenAI if using Azure OpenAI, consider use Workload Identity https://learn.microsoft.com/azure/aks/open-ai-secure-access-quickstart export OPENAI_ORG_ID= # required if using OpenAI uvicorn main:app --host 127.0.0.1 --port 5001 diff --git a/src/ai-service/main.py b/src/ai-service/main.py index db3cbb24..2a10d59f 100644 --- a/src/ai-service/main.py +++ b/src/ai-service/main.py @@ -1,17 +1,27 @@ from fastapi import FastAPI, status from routers.description_generator import description +from routers.image_generator import image from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse import os app = FastAPI(version=os.environ.get("APP_VERSION", "0.1.0")) app.include_router(description) +app.include_router(image) app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) - @app.get("/health", summary="check if server is healthy", operation_id="health") -async def get_products(): +async def get_health(): """ Returns status code 200 """ - return JSONResponse(content={"status": 'ok', "version": app.version}, status_code=status.HTTP_200_OK) + # Initialize the array with "description" + capabilities = ["description"] + + # Check if the environment variable is set + if os.environ.get("AZURE_OPENAI_DALLE_DEPLOYMENT_NAME"): + # If it is, add "image" to the array + capabilities.append("image") + + print("Generative AI capabilities: ", ", ".join(capabilities)) + return JSONResponse(content={"status": 'ok', "version": app.version, "capabilities": capabilities}, status_code=status.HTTP_200_OK) diff --git a/src/ai-service/requirements.txt b/src/ai-service/requirements.txt index 33cf90fc..ace26d02 100644 --- a/src/ai-service/requirements.txt +++ b/src/ai-service/requirements.txt @@ -5,5 +5,8 @@ pytest==7.3.1 httpx pyyaml semantic-kernel==0.4.2.dev0 -azure.identity==1.14.0 -requests==2.31.0 \ No newline at end of file +azure.identity==1.16.0 +requests==2.31.0 + +openai==1.23.2 +pillow==10.3.0 \ No newline at end of file diff --git a/src/ai-service/routers/image_generator.py b/src/ai-service/routers/image_generator.py new file mode 100644 index 00000000..060cbdf1 --- /dev/null +++ b/src/ai-service/routers/image_generator.py @@ -0,0 +1,58 @@ +from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from openai import AzureOpenAI +from fastapi import APIRouter, Request, status +from fastapi.responses import Response, JSONResponse +from typing import Any, List, Dict +import json +import os + +# Define the image API router +image: APIRouter = APIRouter(prefix="/generate", tags=["generate"]) + +# Define the Product class +class Product: + def __init__(self, product: Dict[str, List]) -> None: + self.name: str = product["name"] + self.description: List[str] = product["description"] + +# Define the post_image endpoint +@image.post("/image", summary="Get image for a product", operation_id="getImage") +async def post_image(request: Request) -> JSONResponse: + try: + # Parse the request body and create a Product object + body: dict = await request.json() + product: Product = Product(body) + name: str = product.name + description: List = product.description + + print("Calling OpenAI") + + api_version = os.environ.get("AZURE_OPENAI_API_VERSION") + endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT") + model_deployment_name = os.environ.get("AZURE_OPENAI_DALLE_DEPLOYMENT_NAME") + + token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default") + + client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider, + ) + + result = client.images.generate( + model=model_deployment_name, + prompt="Generate a cute photo realistic image of a product in its packaging in front of a plain background for a product called <"+ name + "> with a description <" + description + "> to be sold in an online pet supply store", + n=1 + ) + + json_response = json.loads(result.model_dump_json()) + + if "error" in str(json_response).lower(): + return Response(content=str(json_response), status_code=status.HTTP_401_UNAUTHORIZED) + print(json_response) + + # Return the image as a JSON response + return JSONResponse(content={"image": json_response["data"][0]["url"]}, status_code=status.HTTP_200_OK) + except Exception as e: + # Return an error message as a JSON response + return JSONResponse(content={"error": str(e)}, status_code=status.HTTP_400_BAD_REQUEST) \ No newline at end of file diff --git a/src/ai-service/test-ai-service.http b/src/ai-service/test-ai-service.http index 457d7ea4..de6b681c 100644 --- a/src/ai-service/test-ai-service.http +++ b/src/ai-service/test-ai-service.http @@ -11,3 +11,13 @@ Content-Type: application/json "name": "Seafarer's Tug Rope", "tags": ["toy","dog"] } + +### Generate product image +POST /generate/image +Host: localhost:5001 +Content-Type: application/json + +{ + "name": "Seafarer's Tug Rope", + "description": "Engage your pup in a game of tug-of-war with the Seafarer's Tug Rope. Made from durable materials, this toy is perfect for interactive playtime and bonding with your furry friend." +} From a9989cae414d85f4f876eea79ef46cf5cc8d098f Mon Sep 17 00:00:00 2001 From: Paul Yu Date: Tue, 23 Apr 2024 10:31:46 -0700 Subject: [PATCH 02/19] feat: adding image generation call to product-service --- src/product-service/src/routes/ai.rs | 38 ++++++++++++++++++- src/product-service/src/startup.rs | 4 ++ src/product-service/test-product-service.http | 11 ++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/product-service/src/routes/ai.rs b/src/product-service/src/routes/ai.rs index b48132cb..70501c61 100644 --- a/src/product-service/src/routes/ai.rs +++ b/src/product-service/src/routes/ai.rs @@ -7,6 +7,7 @@ use std::fmt; use actix_web::{web, Error, HttpResponse, ResponseError}; use crate::startup::AppState; use futures_util::StreamExt; +use serde::{Deserialize, Serialize}; #[derive(Debug)] pub struct ProxyError(reqwest::Error); @@ -29,6 +30,13 @@ impl From for ProxyError { } } +#[derive(Debug, Deserialize, Serialize)] +struct AIHealthResponseBody { + status: String, + version: String, + capabilities: Vec, +} + pub async fn ai_health(data: web::Data) -> Result { let client = reqwest::Client::new(); let ai_service_url = data.settings.ai_service_url.to_owned(); @@ -40,7 +48,7 @@ pub async fn ai_health(data: web::Data) -> Result }; let status = resp.status(); if status.is_success() { - let body: HashMap = resp.json().await.map_err(ProxyError::from)?; + let body: AIHealthResponseBody = resp.json().await.map_err(ProxyError::from)?; let body_json = serde_json::to_string(&body).unwrap(); Ok(HttpResponse::Ok().body(body_json)) } else { @@ -66,6 +74,34 @@ pub async fn ai_generate_description(data: web::Data, mut payload: web } }; + let status = resp.status(); + let body: HashMap = resp.json().await.map_err(ProxyError::from)?; + let body_json = serde_json::to_string(&body).unwrap(); + if status.is_success() { + Ok(HttpResponse::Ok().body(body_json)) + } else { + Ok(HttpResponse::build(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR).body(body_json)) + } +} + +pub async fn ai_generate_image(data: web::Data, mut payload: web::Payload) -> Result { + let mut body = web::BytesMut::new(); + while let Some(chunk) = payload.next().await { + let chunk = chunk?; + body.extend_from_slice(&chunk); + } + let client = reqwest::Client::new(); + let ai_service_url = data.settings.ai_service_url.to_owned(); + let resp = match client.post( ai_service_url + "/generate/image") + .body(body.to_vec()) + .send() + .await { + Ok(resp) => resp, + Err(e) => { + return Ok(HttpResponse::InternalServerError().json(e.to_string())) + } + }; + let status = resp.status(); let body: HashMap = resp.json().await.map_err(ProxyError::from)?; let body_json = serde_json::to_string(&body).unwrap(); diff --git a/src/product-service/src/startup.rs b/src/product-service/src/startup.rs index 40623fbb..dcb56bdb 100644 --- a/src/product-service/src/startup.rs +++ b/src/product-service/src/startup.rs @@ -49,6 +49,10 @@ pub fn run(mut settings: Settings) -> Result { "/ai/generate/description", web::post().to(ai_generate_description), ) + .route( + "/ai/generate/image", + web::post().to(ai_generate_image), + ) }) .listen(listener)? .run(); diff --git a/src/product-service/test-product-service.http b/src/product-service/test-product-service.http index 92529879..44678648 100644 --- a/src/product-service/test-product-service.http +++ b/src/product-service/test-product-service.http @@ -80,3 +80,14 @@ Content-Type: application/json "tags": ["toy","dog"] } + +### Get product image from ai service +POST /ai/generate/image +Host: localhost:3002 +Content-Type: application/json + +{ + "name": "Seafarer's Tug Rope", + "description": "Engage your pup in a game of tug-of-war with the Seafarer's Tug Rope. Made from durable materials, this toy is perfect for interactive playtime and bonding with your furry friend." +} + From c40fb17277a718e9cc718968f57616c45d95fe58 Mon Sep 17 00:00:00 2001 From: Paul Yu Date: Tue, 23 Apr 2024 14:06:46 -0700 Subject: [PATCH 03/19] feat: adding image generation capabilities to store-admin ui --- src/store-admin/nginx.conf | 5 + .../src/components/ProductForm.vue | 183 +++++++++++++++--- src/store-admin/vue.config.js | 34 +++- 3 files changed, 185 insertions(+), 37 deletions(-) diff --git a/src/store-admin/nginx.conf b/src/store-admin/nginx.conf index b5a4a898..a0c5eedc 100644 --- a/src/store-admin/nginx.conf +++ b/src/store-admin/nginx.conf @@ -88,4 +88,9 @@ server { proxy_pass http://product-service:3002/ai/generate/description; proxy_http_version 1.1; } + + location /ai/generate/image { + proxy_pass http://product-service:3002/ai/generate/image; + proxy_http_version 1.1; + } } \ No newline at end of file diff --git a/src/store-admin/src/components/ProductForm.vue b/src/store-admin/src/components/ProductForm.vue index 88715b56..49ee60a5 100644 --- a/src/store-admin/src/components/ProductForm.vue +++ b/src/store-admin/src/components/ProductForm.vue @@ -10,32 +10,50 @@
-
- - -
- -
- - -
- -
- - -
- -
- -