Multimodal Rag Using Generative Search

const {
  data: { payload }
} = await unbody.get
                .imageBlock
                .where({ mimeType: "image/jpeg" })
                .limit(2)
                .select("url", "autoCaption", "alt", "autoOCR", "originalName")
                .generate.fromMany({
                  messages: [
                    {
                      content: "Analyze these images comprehensively. Focus on visual content, caption accuracy, and any text found in the images."
                    },
                    {
                      type: "image",
                      content: "{imageData.urls}"
                    },
                    {
                      content: `Please analyze based on this information:
                        Image Details:
                        {imageData.details}
                        
                        OCR Text Found:
                        {imageData.ocrText}
                        
                        Generated Captions:
                        {imageData.captions}
                        
                        Consider:
                        1. Are the auto-generated captions accurate?
                        2. How does the OCR text relate to the visual content?
                        3. Compare the alt descriptions with what you see.`
                    }
                  ],
                  options: {
                    model: "gpt-4-turbo",
                    vars: [
                      {
                        name: "imageData",
                        formatter: "jq",
                        expression: `{
                          urls: [.[].url],
                          details: map({
                            filename: .originalName,
                            altDescription: .alt
                          }),
                          ocrText: map(.autoOCR),
                          captions: map(.autoCaption)
                        }`
                      }
                    ],
                    maxTokens: 1500,
                    temperature: 0.7
                  }
                })
                .exec();

{
  Get {
    ImageBlock(
      where: {
        operator: Equal, 
        path: "mimeType", 
        valueText: "image/jpeg"
      }
      limit: 2
    ) {
      _additional {
        generate(
          groupedResult: {
            messages: [
              {
                content: "Analyze these images comprehensively. Focus on visual content, caption accuracy, and any text found in the images."
              }, 
              {
                type: "image", 
                content: "{imageData.urls}"
              }, 
              {
                content: "Please analyze based on this information:\n          
                      Image Details:\n          
                      {imageData.details}\n          
                      OCR Text Found:\n          
                      {imageData.ocrText}\n          
                      Generated Captions:\n          
                      {imageData.captions}\n          
                      Consider:\n          
                      1. Are the auto-generated captions accurate?\n          
                      2. How does the OCR text relate to the visual content?\n          
                      3. Compare the alt descriptions with what you see."
              }
            ], 
            options: {
              model: "gpt-4-turbo", 
              vars: [
                {
                  name: "imageData", 
                  formatter: "jq", 
                  expression: "{\n              
                    urls: [.[].url],\n              
                    details: map({\n                
                      filename: .originalName,\n                
                      altDescription: .alt\n              
                    }),\n              
                    ocrText: map(.autoOCR),\n              
                    captions: map(.autoCaption)\n            
                  }"
                }
              ], 
              maxTokens: 1500, 
              temperature: 0.7
            }
          }
        ) {
          error
          groupedResult
          metadata {
            finishReason
            usage {
              inputTokens
              outputTokens
              totalTokens
            }
          }
        }
      }
      url
      autoCaption
      alt
      autoOCR
      originalName
    }
  }
}