Merge pull request #22 from yooper/add_text_corpus

Add text corpus, lexical diversity algorithms and helper methods

Author: yooper

composer.json

@@ -13,7 +13,7 @@
         "psr-4": {
             "TextAnalysis\\": "src/"
 	},
-        "files": ["src/helpers/storage.php", "src/helpers/print.php", "src/helpers/simplified.php"]
+        "files": ["src/helpers/storage.php", "src/helpers/print.php", "src/helpers/simplified.php", "src/helpers/helpers.php"]
     },
     "autoload-dev": {
         "files": ["tests/TestBaseCase.php"]
@@ -24,7 +24,7 @@
         }
     },
     "require" : {
-        "php": ">=5.5",
+        "php": ">=7.0",
         "yooper/stop-words": "^1.0",
         "symfony/console": ">=2.7",
         "camspiers/porter-stemmer": "1.0.*",
@@ -33,6 +33,6 @@
     },
     "require-dev": {
         "phpunit/phpunit": "5.*",
-        "mockery/mockery" : "0.9.7"
+        "mockery/mockery" : "1.0"
     }
 }

src/Collocations/CollocationFinder.php

@@ -27,8 +27,8 @@ public function __construct(array $tokens, $nGramSize = 2)
      */
     public function getCollocations()
     {
-        $nGramTokens = NGramFactory::create($this->tokens, $this->nGramSize);
-        return (new FreqDist($nGramTokens))->getKeyValuesByFrequency();
+        $nGramTokens = ngrams($this->tokens, $this->nGramSize);
+        return freq_dist($nGramTokens)->getKeyValuesByFrequency();
     }
 
     /**
@@ -37,8 +37,8 @@ public function getCollocations()
      */
     public function getCollocationsByPmi()
     {
-        $nGramFreqDist = new FreqDist(NGramFactory::create($this->tokens, $this->nGramSize));
-        $unigramsFreqDist = new FreqDist($this->tokens);
+        $nGramFreqDist = freq_dist(ngrams($this->tokens, $this->nGramSize));
+        $unigramsFreqDist = freq_dist($this->tokens);
 
         $dataSet = [];
         foreach($nGramFreqDist->getKeys() as $nGramToken)

src/Comparisons/LongestCommonSubstringComparison.php

@@ -13,7 +13,7 @@
 class LongestCommonSubstringComparison implements ISimilarity, IDistance
 {
     /**
-     * Using caching to improve preformance on text2 inputs 
+     * Using caching to improve performance on text2 inputs 
      * @var boolean
      */
     protected $useCache = false;

src/Corpus/TextCorpus.php

@@ -0,0 +1,152 @@
+<?php
+
+namespace TextAnalysis\Corpus;
+
+use TextAnalysis\Tokenizers\GeneralTokenizer;
+use TextAnalysis\LexicalDiversity\Naive;
+
+/**
+ * Explore the text corpus
+ * @author yooper
+ */
+class TextCorpus 
+{
+    /**
+     *
+     * @var string
+     */
+    protected $text;
+    
+    /**
+     *
+     * @var array
+     */
+    protected $tokens = [];
+    
+    public function __construct(string $text) 
+    {
+        $this->text = $text;
+    }
+    
+    /**
+     * Returns the original text
+     * @return string
+     */
+    public function getText() : string
+    {
+        return $this->text;
+    }
+    
+    public function getTokens(string $tokenizerClassName = GeneralTokenizer::class) : array
+    {
+        if(empty($this->tokens)) {            
+            $this->tokens = tokenize($this->getText(), $tokenizerClassName);
+        }
+        return $this->tokens;
+    }
+    
+    /**
+     * Return a list of positions that the needs were found in the text
+     * @param array $needles
+     * @return int[]
+     */
+    public function getDispersion(array $needles) : array
+    {
+        $found = array_fill_keys($needles, []);
+        foreach(array_keys($needles) as $needle)
+        {
+            $found[$needle] = $this->findAll($needle);
+        }
+        return $found;
+    }
+    
+    /**
+     * Compute the lexical diversity, the default uses a naive algorithm
+     * @param string $lexicalDiversityClassName
+     * @return float
+     */
+    public function getLexicalDiversity(string $lexicalDiversityClassName = Naive::class) : float
+    {
+        return lexical_diversity($this->getTokens(), $lexicalDiversityClassName);
+    }
+    
+    /**
+     * See https://stackoverflow.com/questions/15737408/php-find-all-occurrences-of-a-substring-in-a-string
+     * @param string $needle
+     * @param int $spacing The amount of space left and right of the found needle
+     * @return array
+     */
+    public function concordance(string $needle, int $spacing = 20) : array
+    {
+        $position = 0;
+        $found = [];
+        $text = trim(preg_replace('/[\s\t\n\r\s]+/', ' ', $this->text));
+        $needleLength = strlen($needle);
+        $textLength = strlen($text);
+        $bufferLength = $needleLength + 2 * $spacing;
+                        
+        while (($position = stripos($text, $needle, $position))!== false) 
+        {
+            $left = max($position - $spacing, 0);                        
+            if($needleLength + $spacing + $position > $textLength) {
+                $tmp = substr($text, $left); 
+            } else { 
+                $tmp = substr($text, $left, $bufferLength);
+            }            
+            $found[] = $tmp;
+            $position += $needleLength;
+        }
+        return $found;
+    }
+    
+    /**
+     * Get percentage of times the needle shows up in the text
+     * @param string $needle
+     * @return float
+     */
+    public function percentage(string $needle) : float
+    {
+        $freqDist = freq_dist($this->getTokens());
+        return $freqDist->getKeyValuesByFrequency()[$needle] / $freqDist->getTotalTokens();
+    }
+    
+    /**
+     * Performs a case insensitive search for the needle
+     * @param string $needle
+     * @return int
+     */
+    public function count(string $needle) : int
+    {
+        return substr_count(strtolower($this->getText()), strtolower($needle));
+    }
+    
+    /**
+     * Return all the position of the needle found in the text
+     * @param string $needle
+     * @return array
+     */
+    public function findAll(string $needle) : array
+    {
+        $lastPos = 0;
+        $positions = [];
+        $needle = strtolower($needle);
+        $text = strtolower($this->getText());
+        $needleLength = strlen($needle);
+        while (($lastPos = stripos($text, $needle, $lastPos))!== false) 
+        {
+            $positions[] = $lastPos;
+            $lastPos += $needleLength;
+        }
+        return $positions;
+    }
+    public function toString()
+    {
+        return $this->text;
+    }
+    
+    public function __destruct() 
+    {
+        unset($this->text);
+        unset($this->tokens);
+    }
+}

src/Indexes/TfIdf.php

@@ -37,7 +37,7 @@ public function __construct(ICollection $collection)
     protected function buildIndex(ICollection $collection)
     {                
         foreach($collection as $id => $document){
-            $freqDist = new FreqDist($document->getDocumentData());
+            $freqDist = freq_dist($document->getDocumentData());
             foreach($freqDist->getKeyValuesByFrequency() as $key => $freq) { 
                 if(!isset($this->idf[$key])) { 
                     $this->idf[$key] = 0;

src/Interfaces/ILexicalDiversity.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace TextAnalysis\Interfaces;
+
+/**
+ * Lexical Diversity Interface
+ * @author developer
+ */
+interface ILexicalDiversity 
+{
+    public function getDiversity(array $tokens) : float;
+}

src/LexicalDiversity/Naive.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace TextAnalysis\LexicalDiversity;
+
+/**
+ * A very simple algorithm for measuring lexical diversity;
+ *
+ * @author yooper
+ */
+class Naive implements \TextAnalysis\Interfaces\ILexicalDiversity
+{
+    public function getDiversity(array $tokens) : float
+    {
+        
+        return count(array_unique( $tokens )) / array_sum( array_map( 'strlen', $tokens) );
+    }
+}

src/LexicalDiversity/YuleI.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace TextAnalysis\LexicalDiversity;
+
+/**
+ * Implementation of Yule's I algorithm
+ * @author yooper
+ */
+class YuleI
+{
+    public function getDiversity(array $tokens) : float
+    {
+        $freq = array_count_values($tokens);        
+        $m1 = array_sum( array_values( $freq));
+        $m2 = array_sum( array_map( function($value){ return $value ** 2; }, array_values($freq)));
+        return ($m1*$m1) / ($m2-$m1);        
+    }   
+}

src/LexicalDiversity/YuleK.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace TextAnalysis\LexicalDiversity;
+
+/**
+ * Description of YuleK
+ *
+ * @author yooper
+ */
+class YuleK extends YuleI implements \TextAnalysis\Interfaces\ILexicalDiversity
+{        
+    public function getDiversity(array $tokens): float 
+    {
+        return 1 / parent::getDiversity($tokens) * 10000;
+    }
+}

src/NGrams/NGramFactory.php

@@ -23,7 +23,7 @@ protected function __construct(){}
      * @param int $nGramSize
      * @return  array return an array of the ngrams
      */
-    static public function create(array $tokens, $nGramSize = self::BIGRAM, $separator = ' ')
+    static public function create(array $tokens, $nGramSize = self::BIGRAM, $separator = ' ') : array
     {
         $separatorLength = strlen($separator);
         $length = count($tokens) - $nGramSize + 1;