update route and prompt (#33)

* update route and prompt

* pretty

* rm log

* rm files

Author: 0xKurt

src/controllers/evaluationController.ts

@@ -300,8 +300,9 @@ const batchPromises = <T>(array: T[], batchSize: number): T[][] => {
 
 export const createLLMEvaluations = async (
   paramsArray: CreateLLMEvaluationParams[]
-): Promise<void> => {
+): Promise<string[]> => {
   const roundCache: Record<string, RoundWithApplications> = {};
+  const failedProjects: string[] = [];
 
   // Split the paramsArray into batches of 10
   const batchedParams = batchPromises(paramsArray, 10);
@@ -310,86 +311,95 @@ export const createLLMEvaluations = async (
     try {
       // Process each batch of promises concurrently
       const evaluationPromises = batch.map(async params => {
-        const evaluationQuestions =
-          params.questions === undefined || params.questions.length === 0
-            ? await evaluationService.getQuestionsByChainAndAlloPoolId(
-                params.chainId,
-                params.alloPoolId
-              )
-            : params.questions;
-
-        if (evaluationQuestions === null || evaluationQuestions.length === 0) {
-          logger.error(
-            'createLLMEvaluations:Failed to get evaluation questions'
-          );
-          throw new Error('Failed to get evaluation questions');
-        }
-
-        let roundMetadata = params.roundMetadata;
-        let applicationMetadata = params.applicationMetadata;
-
-        // Check if the round is already in cache
-        if (roundMetadata == null || applicationMetadata == null) {
-          let round: RoundWithApplications | null;
-
-          // If the round is cached, use it
-          if (roundCache[params.alloPoolId] != null) {
-            round = roundCache[params.alloPoolId];
-            logger.debug(
-              `Using cached round data for roundId: ${params.alloPoolId}`
-            );
-          } else {
-            // Fetch the round and store it in the cache
-            const [error, fetchedRound] = await catchError(
-              indexerClient.getRoundWithApplications({
-                chainId: params.chainId,
-                roundId: params.alloPoolId,
-              })
+        try {
+          const evaluationQuestions =
+            params.questions === undefined || params.questions.length === 0
+              ? await evaluationService.getQuestionsByChainAndAlloPoolId(
+                  params.chainId,
+                  params.alloPoolId
+                )
+              : params.questions;
+
+          if (
+            evaluationQuestions === null ||
+            evaluationQuestions.length === 0
+          ) {
+            logger.error(
+              'createLLMEvaluations:Failed to get evaluation questions'
             );
+            throw new Error('Failed to get evaluation questions');
+          }
 
-            if (error !== undefined || fetchedRound == null) {
-              logger.error('Failed to fetch round with applications');
-              throw new Error('Failed to fetch round with applications');
+          let roundMetadata = params.roundMetadata;
+          let applicationMetadata = params.applicationMetadata;
+
+          // Check if the round is already in cache
+          if (roundMetadata == null || applicationMetadata == null) {
+            let round: RoundWithApplications | null;
+
+            // If the round is cached, use it
+            if (roundCache[params.alloPoolId] != null) {
+              round = roundCache[params.alloPoolId];
+              logger.debug(
+                `Using cached round data for roundId: ${params.alloPoolId}`
+              );
+            } else {
+              // Fetch the round and store it in the cache
+              const [error, fetchedRound] = await catchError(
+                indexerClient.getRoundWithApplications({
+                  chainId: params.chainId,
+                  roundId: params.alloPoolId,
+                })
+              );
+
+              if (error !== undefined || fetchedRound == null) {
+                logger.error('Failed to fetch round with applications');
+                throw new Error('Failed to fetch round with applications');
+              }
+
+              round = fetchedRound;
+              roundCache[params.alloPoolId] = round;
+              logger.info(
+                `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications`
+              );
             }
 
-            round = fetchedRound;
-            roundCache[params.alloPoolId] = round;
-            logger.info(
-              `Fetched and cached round with ID: ${round.id}, which includes ${round.applications.length} applications`
+            const application = round.applications.find(
+              app => app.id === params.alloApplicationId
             );
+            if (application == null) {
+              logger.error(
+                `Application with ID: ${params.alloApplicationId} not found in round`
+              );
+              throw new NotFoundError(
+                `Application with ID: ${params.alloApplicationId} not found in round`
+              );
+            }
+
+            roundMetadata = round.roundMetadata;
+            applicationMetadata = application.metadata;
           }
 
-          const application = round.applications.find(
-            app => app.id === params.alloApplicationId
+          const evaluation = await requestEvaluation(
+            roundMetadata,
+            applicationMetadata,
+            evaluationQuestions
           );
-          if (application == null) {
-            logger.error(
-              `Application with ID: ${params.alloApplicationId} not found in round`
-            );
-            throw new NotFoundError(
-              `Application with ID: ${params.alloApplicationId} not found in round`
-            );
-          }
 
-          roundMetadata = round.roundMetadata;
-          applicationMetadata = application.metadata;
+          await createEvaluation({
+            chainId: params.chainId,
+            alloPoolId: params.alloPoolId,
+            alloApplicationId: params.alloApplicationId,
+            cid: params.cid,
+            evaluator: params.evaluator,
+            summaryInput: evaluation,
+            evaluatorType: EVALUATOR_TYPE.LLM_GPT3,
+          });
+        } catch (error) {
+          // If an error occurs, add the project ID to the failedProjects array
+          failedProjects.push(params.alloApplicationId);
+          throw error;
         }
-
-        const evaluation = await requestEvaluation(
-          roundMetadata,
-          applicationMetadata,
-          evaluationQuestions
-        );
-
-        await createEvaluation({
-          chainId: params.chainId,
-          alloPoolId: params.alloPoolId,
-          alloApplicationId: params.alloApplicationId,
-          cid: params.cid,
-          evaluator: params.evaluator,
-          summaryInput: evaluation,
-          evaluatorType: EVALUATOR_TYPE.LLM_GPT3,
-        });
       });
 
       await Promise.all(evaluationPromises);
@@ -405,4 +415,6 @@ export const createLLMEvaluations = async (
       continue;
     }
   }
+
+  return failedProjects;
 };

src/controllers/poolController.ts

@@ -101,16 +101,34 @@ export const syncPool = async (req: Request, res: Response): Promise<void> => {
 
   // ---- LLM evaluation ----
   // Trigger LLM evaluation for the pool if there are applications without evaluations
-  if (skipEvaluation !== true)
-    await triggerLLMEvaluationByPool(
+  let failedProjects: string[] = [];
+  if (skipEvaluation !== true) {
+    failedProjects = await triggerLLMEvaluationByPool(
       alloPoolId,
       chainId,
       indexerPoolData,
       evaluationQuestions
     );
-  // Log success and respond to the request
-  logger.info('successfully synced pool', pool);
-  res.status(200).json({ message: 'pool synced successfully' });
+  }
+
+  // Check if there are any failed projects and respond accordingly
+  if (failedProjects.length > 0) {
+    logger.info('Pool synced successfully with some projects failing', {
+      pool,
+      failedProjects,
+    });
+    res.status(207).json({
+      success: true,
+      message: 'Pool synced successfully, with some projects failing.',
+      failedProjects,
+    });
+  } else {
+    logger.info('Successfully synced pool', pool);
+    res.status(200).json({
+      success: true,
+      message: 'Pool synced successfully.',
+    });
+  }
 };
 
 const handlePoolEvaluationQuestions = async (
@@ -171,7 +189,7 @@ const triggerLLMEvaluationByPool = async (
   chainId: number,
   indexerPoolData: IndexerRoundWithApplications,
   evaluationQuestions: PromptEvaluationQuestions
-): Promise<void> => {
+): Promise<string[]> => {
   const applicationsWithoutLLM =
     await applicationService.getApplicationsWithoutLLMEvalutionsByAlloPoolId(
       alloPoolId,
@@ -212,5 +230,5 @@ const triggerLLMEvaluationByPool = async (
     logger.warn('Some applications were not found in indexerPoolData');
   }
 
-  await createLLMEvaluations(evaluationParamsArray);
+  return await createLLMEvaluations(evaluationParamsArray);
 };

src/ext/openai/prompt.ts

@@ -39,11 +39,11 @@ const sanitizeAndReduceMetadata = (
 };
 
 const sanitizeRoundMetadata = (metadata: RoundMetadata): string => {
-  return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 1000);
+  return sanitizeAndReduceMetadata(metadata, essentialRoundFields, 50000);
 };
 
 const sanitizeApplicationMetadata = (metadata: object): string => {
-  return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 3000);
+  return sanitizeAndReduceMetadata(metadata, essentialApplicationFields, 50000);
 };
 export const createAiEvaluationPrompt = (
   roundMetadata: RoundMetadata,
@@ -52,7 +52,7 @@ export const createAiEvaluationPrompt = (
 ): string => {
   const sanitizedRoundMetadata = sanitizeRoundMetadata(roundMetadata);
   const sanitizedApplicationMetadata = sanitizeApplicationMetadata(
-    applicationMetadata.application.project
+    applicationMetadata.application
   );
 
   const questionsString = applicationQuestions

src/routes/poolRoutes.ts

@@ -18,31 +18,43 @@ const router = Router();
  *               alloPoolId:
  *                 type: string
  *                 description: The ID of the pool to create
- *                 example: "609"  # Example of poolId
+ *                 example: "609"
  *               chainId:
  *                 type: number
  *                 description: The chain ID associated with the pool
- *                 example: 42161  # Example of chainId (Arbitrum)
+ *                 example: 42161
  *               skipEvaluation:
- *                type: boolean
- *                description: Skip evaluation of the pool
- *                example: true
+ *                 type: boolean
+ *                 description: Skip evaluation of the pool
+ *                 example: true
  *             required:
  *               - alloPoolId
  *               - chainId
  *     responses:
  *       201:
- *         description: Pool created successfully
+ *         description: Pool synced successfully
+ *       207:
+ *         description: Pool synced successfully, with some projects failing
+ *         content:
+ *           application/json:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 success:
+ *                   type: boolean
+ *                   example: true
+ *                 message:
+ *                   type: string
+ *                   example: Pool synced successfully, with some projects failing.
+ *                 failedProjects:
+ *                   type: array
+ *                   items:
+ *                     type: string
+ *                     example: "0xprojectId1"
  *       400:
  *         description: Invalid poolId or chainId format
  *       500:
  *         description: Internal server error
- *     examples:
- *       application/json:
- *         - value:
- *             alloPoolId: "609"
- *             chainId: "42161"
- *             skipEvaluation: true
  */
 router.post('/', syncPool);