added limit on number of fields in record (#166)

Author: oasisk

coverage.sh

@@ -17,9 +17,9 @@ echo "line_cov $line_cov"
 
 # enable threshold
 #COVERAGE_THRESHOLD=80
-FUNC_COV_THRESHOLD=40
-LINE_COV_THRESHOLD=35
-REGION_COV_THRESHOLD=25
+FUNC_COV_THRESHOLD=60
+LINE_COV_THRESHOLD=53
+REGION_COV_THRESHOLD=40
 
 # clean up
 # find ./target -name llvm-cov-target -type d|xargs rm -fR

src/infra/config.rs

@@ -196,6 +196,8 @@ pub struct Limit {
     pub hb_interval: i64,
     // no need set by environment
     pub cpu_num: usize,
+    #[env_config(name = "ZO_COLS_PER_RECORD_LIMIT")]
+    pub req_cols_per_record_limit: usize,
 }
 
 #[derive(Clone, Debug, EnvConfig)]
@@ -316,6 +318,10 @@ pub fn init() -> Config {
         panic!("data path config error: {}", e);
     }
 
+    if cfg.limit.req_cols_per_record_limit == 0 {
+        cfg.limit.req_cols_per_record_limit = 1000;
+    }
+
     // check memeory cache
     if let Err(e) = check_memory_cache_config(&mut cfg) {
         panic!("memory cache config error: {}", e);

src/service/schema.rs

@@ -202,6 +202,10 @@ pub async fn check_for_schema(
         return (true, None);
     }
 
+    if inferred_schema.fields.len() > CONFIG.limit.req_cols_per_record_limit {
+        return (false, None);
+    }
+
     if schema == Schema::empty() {
         stream_schema_map.insert(stream_name.to_string(), inferred_schema.clone());
         db::schema::set(