[DRAFT][DO NOT MERGE] Add helpers

Gemfile

@@ -7,6 +7,7 @@ gemspec
 
 gem 'rake', '~> 12.0'
 gem 'rspec', '~> 3.0'
+gem 'activesupport'
 group :development, :test do
   gem 'rubocop'
   gem 'webmock', '~> 3.0'

Gemfile.lock

@@ -9,47 +9,66 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.2.0)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.2)
+    base64 (0.2.0)
     bigdecimal (3.1.8)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
     crack (1.0.0)
       bigdecimal
       rexml
     diff-lcs (1.5.1)
-    hashdiff (1.1.0)
+    drb (2.2.1)
+    hashdiff (1.1.1)
+    i18n (1.14.5)
+      concurrent-ruby (~> 1.0)
     json (2.7.2)
     language_server-protocol (3.17.0.3)
+    logger (1.6.0)
+    minitest (5.25.1)
     net-http (0.1.1)
       net-protocol
       uri
     net-protocol (0.2.2)
       timeout
-    parallel (1.25.1)
-    parser (3.3.4.0)
+    parallel (1.26.3)
+    parser (3.3.4.2)
       ast (~> 2.4.1)
       racc
-    public_suffix (5.1.1)
+    public_suffix (6.0.1)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (12.3.3)
     regexp_parser (2.9.2)
-    rexml (3.3.2)
+    rexml (3.3.5)
       strscan
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.1)
+    rspec-expectations (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    rubocop (1.65.0)
+    rubocop (1.65.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -60,11 +79,14 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.3)
+    rubocop-ast (1.32.1)
       parser (>= 3.3.1.0)
     ruby-progressbar (1.13.0)
+    securerandom (0.3.1)
     strscan (3.1.0)
     timeout (0.4.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
     uri (0.13.0)
     webmock (3.23.1)
@@ -76,6 +98,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport
   rake (~> 12.0)
   rspec (~> 3.0)
   rspec-llama!

examples/example_spec.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  WebMock.allow_net_connect!
+
+  config.api_endpoint = 'http://localhost:3000'
+  config.auth_endpoint = 'http://localhost:3000/users/sign_in'
+  config.api_creds = { user: { email: '***email***', password: '***password***' } }
+
+  config.include Rspec::Llama::Helpers
+end
+
+RSpec.describe 'Llama Rspec flow' do
+  let(:model_version_name) { 'Version 1' }
+
+  context 'when we want to fetch resources' do
+    before do
+      use_model('LLAMA_C', 'Vesion C')
+      build_prompt('pt_c', 'Is Minsk the capital of Belarus?')
+      build_assertion('1_ass', 'Yes')
+      use_prompt('pt_c')
+      use_assertion('1_ass')
+    end
+
+    it 'should fetch model & model_version' do
+      expect(settled_model['name']).to eq('LLAMA_C')
+      expect(settled_model_version['build_name']).to eq('Vesion C')
+      expect(settled_prompt['name']).to eq('pt_c')
+      expect(settled_assertion['name']).to eq('1_ass')
+    end
+  end
+
+  context 'when we want to create resources(prompt, assertion)' do
+    before do
+      use_model('LLAMA_C', 'Vesion C')
+      build_prompt('pt_s', 'Is Minsk the capital of Belarus?')
+      build_assertion('ass_1', 'Yes')
+    end
+
+    it 'should create prompt & assertion' do
+      expect(settled_prompt['name']).to eq('pt_s')
+      expect(settled_assertion['name']).to eq('ass_1')
+    end
+  end
+
+  context 'when we want to set resources' do
+    before do
+      use_model('LLAMA', model_version_name)
+      build_prompt('pt_2', 'What is the capital of France?')
+      build_assertion('ass_2', 'No')
+    end
+
+    it 'should set model, prompt, assertion' do
+      expect(settled_model['name']).to eq('LLAMA')
+      expect(settled_model_version['build_name']).to eq(model_version_name)
+      expect(settled_prompt['name']).to eq('pt_2')
+      expect(settled_assertion['name']).to eq('ass_2')
+    end
+  end
+
+  context 'when we want to create resources' do
+    before do
+      use_model('LLAMA_C', 'Vesion C')
+      build_prompt('pt_c', 'What is the capital of France?')
+      build_assertion('ass_c', 'Paris')
+    end
+
+    it 'should create model, model_version, prompt, assertion' do
+      expect(settled_model['name']).to eq('LLAMA_C')
+      expect(settled_model_version['build_name']).to eq('Vesion C')
+      expect(settled_prompt['name']).to eq('pt_c')
+      expect(settled_assertion['name']).to eq('ass_c')
+    end
+  end
+
+  describe '#execute_test_run' do
+    before do
+      use_model(model_name, model_version_name)
+      use_prompt('pt_2')
+      use_assertion('ass_2')
+    end
+
+    let(:action) { execute_test_run }
+
+    context 'model version 1' do
+      let(:model_name) { 'LLAMA' }
+      let(:model_version_name) { 'Version 2' }
+      it 'should return a completed status' do
+        action
+        expect(test_run).to be_successful
+      end
+    end
+
+    context 'model version 2' do
+      let(:model_name) { 'LLAMA' }
+      let(:model_version_name) { 'Version 2' }
+
+      it 'should return a completed status' do
+        action
+        expect(test_run).to be_successful
+      end
+
+      context 'create a new prompt & assertion' do
+        before do
+          build_prompt('pt_2', 'what is the capital of france?')
+          build_assertion('ass_2', 'paris')
+        end
+
+        it 'should return a completed status' do
+          action
+          expect(test_run).to be_failed
+        end
+      end
+
+      context 'with more manual testing' do
+        before do
+          build_prompt('pt_2', 'what is the capital of france?')
+          build_assertion('ass_2', 'paris')
+        end
+
+        it 'should return a completed status' do
+          action
+          expect(settled_test_results.last['status']).to eq('completed')
+          expect(settled_assertion_results.first['state']).to eq('failed')
+        end
+      end
+    end
+  end
+end

lib/rspec/llama.rb

@@ -8,3 +8,4 @@
 require_relative 'llama/configuration'
 require_relative 'llama/version'
 require_relative 'llama/helpers'
+require_relative 'llama/support/test_run'

lib/rspec/llama/api_client.rb

@@ -9,38 +9,108 @@ def initialize(endpoint, creds, auth_endpoint)
         @endpoint = endpoint
         @creds = creds
         @auth_endpoint = auth_endpoint
+
+        set_token
       end
 
       def authenticate
         uri = URI(auth_endpoint)
-        req = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
-        req.body = creds.to_json
+        api_execute(:post, uri, creds)
+      end
 
-        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
-          http.request(req)
-        end
+      def fetch_all_models
+        uri = URI("#{endpoint}/models.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_all_prompts
+        uri = URI("#{endpoint}/prompts.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_all_assertions
+        uri = URI("#{endpoint}/assertions.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_all_model_versions(model_id)
+        uri = URI("#{endpoint}/models/#{model_id}/model_versions.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_assertion_results(test_result_id)
+        uri = URI("#{endpoint}/test_results/#{test_result_id}/assertion_results.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_model(id)
+        uri = URI("#{endpoint}/models/#{id}.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_test_run(test_run_id)
+        uri = URI("#{endpoint}/test_runs/#{test_run_id}.json")
+        api_execute(:get, uri)
+      end
 
-        response['Authorization'].gsub('Bearer ', '')
+      def fetch_test_model_version_run(test_model_version_run_id)
+        uri = URI("#{endpoint}/test_model_version_runs/#{test_model_version_run_id}.json")
+        api_execute(:get, uri)
+      end
+
+      def fetch_test_result(test_result_id)
+        uri = URI("#{endpoint}/test_results/#{test_result_id}.json")
+        api_execute(:get, uri)
+      end
+
+      def create_model(**opts)
+        uri = URI("#{endpoint}/models.json")
+        api_execute(:post, uri, opts)
+      end
+
+      def create_model_version(model_id, opts)
+        uri = URI("#{endpoint}/models/#{model_id}/model_versions.json")
+        api_execute(:post, uri, opts)
+      end
+
+      def create_prompt(opts)
+        params = { prompt: { value: opts[:value], name: opts[:name] } }
+        uri = URI("#{endpoint}/prompts.json")
+        api_execute(:post, uri, params)
+      end
+
+      def create_assertion(opts)
+        uri = URI("#{endpoint}/assertions.json")
+        api_execute(:post, uri, opts)
       end
 
       def execute_test_run(params)
         uri = URI("#{endpoint}/test_runs.json")
-        req = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
-        req.body = params.to_json
-        req['Authorization'] = "Bearer #{token}"
+        api_execute(:post, uri, params)
+      end
+
+      private
+
+      def api_execute(method, uri, params = nil)
+        is_auth_req = uri.to_s.include?('sign_in')
+
+        req = Net::HTTP.const_get(method.capitalize).new(uri, 'Content-Type' => 'application/json')
+        req['Authorization'] = "Bearer #{token}" unless is_auth_req
+        req.body = params.to_json if params
 
         response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
           http.request(req)
         end
 
-        JSON.parse(response.body)
+        is_auth_req ? response : JSON.parse(response.body)
+      rescue StandardError => e
+        # should be implemented
       end
 
-      private
-
       def token
-        @token ||= authenticate
+        @token ||= authenticate['Authorization'].gsub('Bearer ', '')
       end
+      alias_method :set_token, :token
     end
   end
 end

lib/rspec/llama/helpers.rb

@@ -1,11 +1,19 @@
 # frozen_string_literal: true
 
+require "active_support/all"
+
+require_relative 'helpers/resource_handler'
+require_relative 'helpers/executor'
+require_relative 'helpers/base'
+require_relative 'helpers/errors'
+
 module Rspec
   module Llama
     module Helpers
-      def execute_test_run(test_id)
-        Rspec::Llama.api_client.execute_test_run(test_id)
-      end
+      include Base
+      include ResourceHandler
+      include Executor
+      include Errors
     end
   end
 end