Allow AI to change layers/effects/scene and some game properties (#7799)

Author: 4ian

Core/GDCore/Extensions/Metadata/MetadataProvider.h

@@ -277,6 +277,10 @@ class GD_CORE_API MetadataProvider {
     return &metadata == &badObjectInfo;
   }
 
+  static bool IsBadEffectMetadata(const gd::EffectMetadata& metadata) {
+    return &metadata == &badEffectMetadata;
+  }
+
   virtual ~MetadataProvider();
 
  private:

Extensions/3D/JsExtension.js

@@ -1908,7 +1908,9 @@ module.exports = {
         .addEffect('AmbientLight')
         .setFullName(_('Ambient light'))
         .setDescription(
-          _('A light that illuminates all objects from every direction.')
+          _(
+            'A light that illuminates all objects from every direction. Often used along with a Directional light (though a Hemisphere light can be used instead of an Ambient light).'
+          )
         )
         .markAsNotWorkingForObjects()
         .markAsOnlyWorkingFor3D()
@@ -1929,7 +1931,11 @@ module.exports = {
       const effect = extension
         .addEffect('DirectionalLight')
         .setFullName(_('Directional light'))
-        .setDescription(_('A very far light source like the sun.'))
+        .setDescription(
+          _(
+            "A very far light source like the sun. This is the light to use for casting shadows for 3D objects (other lights won't emit shadows). Often used along with a Hemisphere light."
+          )
+        )
         .markAsNotWorkingForObjects()
         .markAsOnlyWorkingFor3D()
         .addIncludeFile('Extensions/3D/DirectionalLight.js');
@@ -2013,7 +2019,7 @@ module.exports = {
         .setFullName(_('Hemisphere light'))
         .setDescription(
           _(
-            'A light that illuminates objects from every direction with a gradient.'
+            'A light that illuminates objects from every direction with a gradient. Often used along with a Directional light.'
           )
         )
         .markAsNotWorkingForObjects()

GDevelop.js/Bindings/Bindings.idl

@@ -1143,6 +1143,7 @@ interface LayersContainer {
     boolean HasLayerNamed([Const] DOMString name);
     void RemoveLayer([Const] DOMString name);
     unsigned long GetLayersCount();
+    unsigned long GetLayerPosition([Const] DOMString name);
     void SwapLayers(unsigned long firstLayerIndex, unsigned long secondLayerIndex);
     void MoveLayer(unsigned long oldIndex, unsigned long newIndex);
     void SerializeLayersTo([Ref] SerializerElement element);
@@ -2932,6 +2933,7 @@ interface MetadataProvider {
     boolean STATIC_IsBadInstructionMetadata([Const, Ref] InstructionMetadata metadata);
     boolean STATIC_IsBadBehaviorMetadata([Const, Ref] BehaviorMetadata metadata);
     boolean STATIC_IsBadObjectMetadata([Const, Ref] ObjectMetadata metadata);
+    boolean STATIC_IsBadEffectMetadata([Const, Ref] EffectMetadata metadata);
 };
 
 enum ProjectDiagnostic_ErrorType {

GDevelop.js/Bindings/Wrapper.cpp

@@ -606,6 +606,7 @@ typedef std::vector<gd::PropertyDescriptorChoice> VectorPropertyDescriptorChoice
 #define STATIC_IsBadInstructionMetadata IsBadInstructionMetadata
 #define STATIC_IsBadBehaviorMetadata IsBadBehaviorMetadata
 #define STATIC_IsBadObjectMetadata IsBadObjectMetadata
+#define STATIC_IsBadEffectMetadata IsBadEffectMetadata
 
 #define STATIC_RenameObjectInEvents RenameObjectInEvents
 #define STATIC_RemoveObjectInEvents RemoveObjectInEvents

GDevelop.js/types.d.ts

@@ -941,6 +941,7 @@ export class LayersContainer extends EmscriptenObject {
   hasLayerNamed(name: string): boolean;
   removeLayer(name: string): void;
   getLayersCount(): number;
+  getLayerPosition(name: string): number;
   swapLayers(firstLayerIndex: number, secondLayerIndex: number): void;
   moveLayer(oldIndex: number, newIndex: number): void;
   serializeLayersTo(element: SerializerElement): void;
@@ -2109,6 +2110,7 @@ export class MetadataProvider extends EmscriptenObject {
   static isBadInstructionMetadata(metadata: InstructionMetadata): boolean;
   static isBadBehaviorMetadata(metadata: BehaviorMetadata): boolean;
   static isBadObjectMetadata(metadata: ObjectMetadata): boolean;
+  static isBadEffectMetadata(metadata: EffectMetadata): boolean;
 }
 
 export class ProjectDiagnostic extends EmscriptenObject {

GDevelop.js/types/gdlayerscontainer.js

@@ -7,6 +7,7 @@ declare class gdLayersContainer {
   hasLayerNamed(name: string): boolean;
   removeLayer(name: string): void;
   getLayersCount(): number;
+  getLayerPosition(name: string): number;
   swapLayers(firstLayerIndex: number, secondLayerIndex: number): void;
   moveLayer(oldIndex: number, newIndex: number): void;
   serializeLayersTo(element: gdSerializerElement): void;

GDevelop.js/types/gdmetadataprovider.js

@@ -26,6 +26,7 @@ declare class gdMetadataProvider {
   static isBadInstructionMetadata(metadata: gdInstructionMetadata): boolean;
   static isBadBehaviorMetadata(metadata: gdBehaviorMetadata): boolean;
   static isBadObjectMetadata(metadata: gdObjectMetadata): boolean;
+  static isBadEffectMetadata(metadata: gdEffectMetadata): boolean;
   delete(): void;
   ptr: number;
 };

newIDE/app/src/AiGeneration/AskAiEditorContainer.js

@@ -5,6 +5,7 @@ import { I18n } from '@lingui/react';
 import {
   type RenderEditorContainerPropsWithRef,
   type SceneEventsOutsideEditorChanges,
+  type InstancesOutsideEditorChanges,
 } from '../MainFrame/EditorContainers/BaseEditor';
 import { type ObjectWithContext } from '../ObjectsList/EnumerateObjects';
 import Paper from '../UI/Paper';
@@ -67,6 +68,7 @@ const useProcessFunctionCalls = ({
   getEditorFunctionCallResults,
   addEditorFunctionCallResults,
   onSceneEventsModifiedOutsideEditor,
+  onInstancesModifiedOutsideEditor,
   onExtensionInstalled,
 }: {|
   i18n: I18nType,
@@ -85,6 +87,9 @@ const useProcessFunctionCalls = ({
   onSceneEventsModifiedOutsideEditor: (
     changes: SceneEventsOutsideEditorChanges
   ) => void,
+  onInstancesModifiedOutsideEditor: (
+    changes: InstancesOutsideEditorChanges
+  ) => void,
   onExtensionInstalled: (extensionNames: Array<string>) => void,
 |}) => {
   const { ensureExtensionInstalled } = useEnsureExtensionInstalled({
@@ -159,6 +164,7 @@ const useProcessFunctionCalls = ({
           });
         },
         onSceneEventsModifiedOutsideEditor,
+        onInstancesModifiedOutsideEditor,
         ensureExtensionInstalled,
         searchAndInstallAsset,
       });
@@ -179,6 +185,7 @@ const useProcessFunctionCalls = ({
       searchAndInstallAsset,
       generateEvents,
       onSceneEventsModifiedOutsideEditor,
+      onInstancesModifiedOutsideEditor,
       triggerSendEditorFunctionCallResults,
       editorCallbacks,
     ]
@@ -349,6 +356,9 @@ type Props = {|
   onSceneEventsModifiedOutsideEditor: (
     changes: SceneEventsOutsideEditorChanges
   ) => void,
+  onInstancesModifiedOutsideEditor: (
+    changes: InstancesOutsideEditorChanges
+  ) => void,
   onExtensionInstalled: (extensionNames: Array<string>) => void,
   initialMode: 'chat' | 'agent' | null,
   initialAiRequestId: string | null,
@@ -372,6 +382,9 @@ export type AskAiEditorInterface = {|
   onSceneEventsModifiedOutsideEditor: (
     changes: SceneEventsOutsideEditorChanges
   ) => void,
+  onInstancesModifiedOutsideEditor: (
+    changes: InstancesOutsideEditorChanges
+  ) => void,
   startOrOpenChat: ({|
     mode: 'chat' | 'agent',
     aiRequestId: string | null,
@@ -401,6 +414,7 @@ export const AskAiEditor = React.memo<Props>(
         onCreateProjectFromExample,
         onOpenLayout,
         onSceneEventsModifiedOutsideEditor,
+        onInstancesModifiedOutsideEditor,
         onExtensionInstalled,
         initialMode,
         initialAiRequestId,
@@ -518,6 +532,7 @@ export const AskAiEditor = React.memo<Props>(
         onSceneObjectEdited: noop,
         onSceneObjectsDeleted: noop,
         onSceneEventsModifiedOutsideEditor: noop,
+        onInstancesModifiedOutsideEditor: noop,
         startOrOpenChat: onStartOrOpenChat,
       }));
 
@@ -655,6 +670,7 @@ export const AskAiEditor = React.memo<Props>(
                 fileMetadata,
                 storageProviderName,
                 mode,
+                toolsVersion: 'v2',
                 aiConfiguration: {
                   presetId: aiConfigurationPresetId,
                 },
@@ -944,6 +960,7 @@ export const AskAiEditor = React.memo<Props>(
         getEditorFunctionCallResults,
         addEditorFunctionCallResults,
         onSceneEventsModifiedOutsideEditor,
+        onInstancesModifiedOutsideEditor,
         i18n,
         onExtensionInstalled,
       });
@@ -1044,6 +1061,9 @@ export const renderAskAiEditorContainer = (
         onSceneEventsModifiedOutsideEditor={
           props.onSceneEventsModifiedOutsideEditor
         }
+        onInstancesModifiedOutsideEditor={
+          props.onInstancesModifiedOutsideEditor
+        }
         onExtensionInstalled={props.onExtensionInstalled}
         initialMode={
           (props.extraEditorProps && props.extraEditorProps.mode) || null

newIDE/app/src/EditorFunctions/EditorFunctionCallRunner.js

@@ -10,6 +10,7 @@ import {
   type AssetSearchAndInstallOptions,
   type AssetSearchAndInstallResult,
   type SceneEventsOutsideEditorChanges,
+  type InstancesOutsideEditorChanges,
 } from '.';
 
 export type EditorFunctionCallResult =
@@ -39,6 +40,9 @@ export type ProcessEditorFunctionCallsOptions = {|
   onSceneEventsModifiedOutsideEditor: (
     changes: SceneEventsOutsideEditorChanges
   ) => void,
+  onInstancesModifiedOutsideEditor: (
+    changes: InstancesOutsideEditorChanges
+  ) => void,
   ensureExtensionInstalled: (options: {|
     extensionName: string,
   |}) => Promise<void>,
@@ -53,6 +57,7 @@ export const processEditorFunctionCalls = async ({
   editorCallbacks,
   generateEvents,
   onSceneEventsModifiedOutsideEditor,
+  onInstancesModifiedOutsideEditor,
   ignore,
   ensureExtensionInstalled,
   searchAndInstallAsset,
@@ -136,6 +141,7 @@ export const processEditorFunctionCalls = async ({
           args,
           generateEvents,
           onSceneEventsModifiedOutsideEditor,
+          onInstancesModifiedOutsideEditor,
           ensureExtensionInstalled,
           searchAndInstallAsset,
         }

newIDE/app/src/EditorFunctions/SimplifiedProject/ExtensionSummary.js

@@ -32,6 +32,12 @@ export type ExpressionSummary = {|
   relevantForSceneEvents?: boolean,
 |};
 
+export type PropertySummary = {|
+  name: string,
+  description: string,
+  type: string,
+|};
+
 export type ObjectSummary = {|
   name: string,
   fullName: string,
@@ -51,6 +57,17 @@ export type BehaviorSummary = {|
   expressions: Array<ExpressionSummary>,
 |};
 
+export type EffectSummary = {|
+  name: string,
+  fullName: string,
+  description: string,
+  notWorkingForObjects: boolean,
+  onlyWorkingFor2D: boolean,
+  onlyWorkingFor3D: boolean,
+  unique: boolean,
+  properties: Array<PropertySummary>,
+|};
+
 export type ExtensionSummary = {|
   extensionName: string,
   extensionFullName: string,
@@ -60,6 +77,7 @@ export type ExtensionSummary = {|
   freeExpressions: Array<ExpressionSummary>,
   objects: { [string]: ObjectSummary },
   behaviors: { [string]: BehaviorSummary },
+  effects: { [string]: EffectSummary },
 |};
 
 const normalizeType = (parameterType: string) => {
@@ -102,6 +120,29 @@ const getParameterSummary = (
   return parameterSummary;
 };
 
+const getPropertySummary = (
+  propertyName: string,
+  property: gdPropertyDescriptor
+) => {
+  return {
+    name: propertyName,
+    description: property.getDescription(),
+    type: property.getType(),
+  };
+};
+
+const getPropertiesSummary = (
+  propertiesMetadata: gdMapStringPropertyDescriptor
+) => {
+  return propertiesMetadata
+    .keys()
+    .toJSArray()
+    .map(propertyName => {
+      const property = propertiesMetadata.get(propertyName);
+      return getPropertySummary(propertyName, property);
+    });
+};
+
 export const buildExtensionSummary = ({
   gd,
   extension,
@@ -111,6 +152,7 @@ export const buildExtensionSummary = ({
 }): ExtensionSummary => {
   const objects: { [string]: ObjectSummary } = {};
   const behaviors: { [string]: BehaviorSummary } = {};
+  const effects: { [string]: EffectSummary } = {};
 
   const generateInstructionsSummaries = ({
     instructionsMetadata,
@@ -254,6 +296,27 @@ export const buildExtensionSummary = ({
 
       behaviors[behaviorType] = behaviorSummary;
     });
+  extension
+    .getExtensionEffectTypes()
+    .toJSArray()
+    .forEach(effectType => {
+      const effectMetadata = extension.getEffectMetadata(effectType);
+      if (gd.MetadataProvider.isBadEffectMetadata(effectMetadata)) {
+        return;
+      }
+      const effectSummary: EffectSummary = {
+        name: effectMetadata.getType(),
+        fullName: effectMetadata.getFullName(),
+        description: effectMetadata.getDescription(),
+        notWorkingForObjects: effectMetadata.isMarkedAsNotWorkingForObjects(),
+        onlyWorkingFor2D: effectMetadata.isMarkedAsOnlyWorkingFor2D(),
+        onlyWorkingFor3D: effectMetadata.isMarkedAsOnlyWorkingFor3D(),
+        unique: effectMetadata.isMarkedAsUnique(),
+        properties: getPropertiesSummary(effectMetadata.getProperties()),
+      };
+
+      effects[effectType] = effectSummary;
+    });
 
   return {
     extensionName: extension.getName(),
@@ -275,5 +338,6 @@ export const buildExtensionSummary = ({
     ],
     objects,
     behaviors,
+    effects,
   };
 };