Merge pull request #252 from acrellin/upload_precomputed_features

Allow uploading of pre-computed featuresets

Author: stefanv

cesium_app/app_server.py

@@ -15,6 +15,7 @@
     ProjectHandler,
     DatasetHandler,
     FeatureHandler,
+    PrecomputedFeaturesHandler,
     ModelHandler,
     PredictionHandler,
     FeatureListHandler,
@@ -58,6 +59,7 @@ def make_app(cfg, baselayer_handlers, baselayer_settings):
         (r'/dataset(/.*)?', DatasetHandler),
         (r'/features(/[0-9]+)?', FeatureHandler),
         (r'/features/([0-9]+)/(download)', FeatureHandler),
+        (r'/precomputed_features(/.*)?', PrecomputedFeaturesHandler),
         (r'/models(/[0-9]+)?', ModelHandler),
         (r'/models/([0-9]+)/(download)', ModelHandler),
         (r'/predictions(/[0-9]+)?', PredictionHandler),

cesium_app/handlers/__init__.py

@@ -10,3 +10,4 @@
 from .plot_features import PlotFeaturesHandler
 from .prediction import PredictionHandler, PredictRawDataHandler
 from .sklearn_models import SklearnModelsHandler
+from .feature import PrecomputedFeaturesHandler

cesium_app/handlers/feature.py

@@ -13,6 +13,7 @@
 from os.path import join as pjoin
 import uuid
 import datetime
+from io import StringIO
 import pandas as pd
 
 
@@ -99,6 +100,7 @@ async def post(self):
         fset = Featureset(name=featureset_name,
                           file_uri=fset_path,
                           project=dataset.project,
+                          dataset=dataset,
                           features_list=features_to_use,
                           custom_features_script=None)
         DBSession().add(fset)
@@ -138,3 +140,45 @@ def delete(self, featureset_id):
     def put(self, featureset_id):
         f = Featureset.get_if_owned_by(featureset_id, self.current_user)
         self.error("Functionality for this endpoint is not yet implemented.")
+
+
+class PrecomputedFeaturesHandler(BaseHandler):
+    @auth_or_token
+    def post(self):
+        data = self.get_json()
+        if data['datasetID'] not in [None, 'None']:
+            dataset = Dataset.query.filter(Dataset.id == data['datasetID']).one()
+        else:
+            dataset = None
+        current_project = Project.get_if_owned_by(data['projectID'],
+                                                  self.current_user)
+        feature_data = StringIO(data['dataFile']['body'])
+        fset = pd.read_csv(feature_data, index_col=0, header=[0, 1])
+        if 'labels' in fset:
+            labels = fset.pop('labels').values.ravel()
+            if labels.dtype == 'O':
+                labels = [str(label) for label in labels]
+        else:
+            labels = [None]
+        fset_path = pjoin(
+            self.cfg['paths:features_folder'],
+            '{}_{}.npz'.format(uuid.uuid4(), data['dataFile']['name']))
+
+        featurize.save_featureset(fset, fset_path, labels=labels)
+
+        # Meta-features will have channel values of an empty string or a string
+        # beginning with 'Unnamed:'
+        features_list = [el[0] for el in fset.columns.tolist() if
+                         (el[1] != '' and not el[1].startswith('Unnamed:'))]
+
+        featureset = Featureset(name=data['featuresetName'],
+                                file_uri=fset_path,
+                                project=current_project,
+                                dataset=dataset,
+                                features_list=features_list,
+                                finished=datetime.datetime.now(),
+                                custom_features_script=None)
+        DBSession().add(featureset)
+        DBSession().commit()
+
+        self.success(featureset, 'cesium/FETCH_FEATURESETS')

cesium_app/models.py

@@ -26,6 +26,7 @@ class Dataset(Base):
     project_id = sa.Column(sa.ForeignKey('projects.id', ondelete='CASCADE'),
                            nullable=False, index=True)
     project = relationship('Project', back_populates='datasets')
+    featureset = relationship('Featureset', back_populates='dataset')
     files = relationship('DatasetFile', backref='dataset', cascade='all')
 
     def display_info(self):
@@ -66,15 +67,15 @@ class Featureset(Base):
     project_id = sa.Column(sa.ForeignKey('projects.id', ondelete='CASCADE'),
                            nullable=False, index=True)
     project = relationship('Project', back_populates='featuresets')
+    dataset_id = sa.Column(sa.ForeignKey('datasets.id'))
+    dataset = relationship('Dataset')
     name = sa.Column(sa.String(), nullable=False)
     features_list = sa.Column(sa.ARRAY(sa.VARCHAR()), nullable=False, index=True)
     custom_features_script = sa.Column(sa.String())
     file_uri = sa.Column(sa.String(), nullable=True, index=True)
     task_id = sa.Column(sa.String())
     finished = sa.Column(sa.DateTime)
 
-    project = relationship('Project')
-
 
 class Model(Base):
     project_id = sa.Column(sa.ForeignKey('projects.id', ondelete='CASCADE'),

cesium_app/tests/data/downloaded_cesium_featureset.csv

@@ -0,0 +1,9 @@
+feature,amplitude,flux_percentile_ratio_mid20,flux_percentile_ratio_mid35,flux_percentile_ratio_mid50,flux_percentile_ratio_mid65,flux_percentile_ratio_mid80,max_slope,maximum,median,median_absolute_deviation,minimum,percent_amplitude,percent_beyond_1_std,percent_close_to_median,percent_difference_flux_percentile,period_fast,qso_log_chi2_qsonu,qso_log_chi2nuNULL_chi2nu,skew,std,stetson_j,stetson_k,weighted_average,meta1,meta2,meta3,labels
+channel,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,,,,
+217801,3.4145,0.026237634586002364,0.058162853124671635,0.10309627831781239,0.17599090611586052,0.41575052067669566,24.652777777737825,14.626,12.42,0.9030000000000005,7.797,69.66429004252072,0.27001862197392923,0.3649906890130354,23.06668928889701,375.6510827444723,4.230919840093198,2.671853833598369,-0.9236797091091277,1.4663914712199768,11.369853415776587,0.9189839436611995,12.040512767456608,0.18073430690900003,0.548427238218,0.18795623725299998,Mira
+224635,0.5305000000000004,0.20530614882878542,0.37706209346535463,0.5453768817756192,0.7330743740788521,0.9166330384370994,17.90927021704068,8.768,8.2,0.18400000000000105,7.707,0.5747078772898464,0.4177396280400572,0.2832618025751073,0.7218041519921866,13.451012652036399,3.2134838274248385,-0.0525010150070849,-0.3398583440079825,0.23517859967140312,1.4786720429259577,1.0411229015075227,8.14368000185732,0.330610932539,0.77316026008,0.0952391836803,Classical_Cepheid
+232798,1.9050000000000002,0.18733902825629659,0.3230574524254608,0.45220729741608706,0.6054170290823713,0.8292614702651346,49.87593052038574,9.179,7.048,0.7784999999999997,5.369,3.6945972728154928,0.3796875,0.2390625,3.3066371717225116,149.38740734028093,4.138293582989955,1.8021678710880233,0.15731166528082127,0.9603469186752173,8.140710283210836,1.0595606571990508,7.093264577381519,0.8972212196189999,0.6016976582729999,0.587206038094,Mira
+235913,0.2919999999999998,0.2143858659023139,0.372513966154211,0.5364876814022899,0.7205289635708764,0.8492495259235322,21.57676348499049,10.522,10.243,0.10899999999999999,9.938,0.3243415351946631,0.39611650485436894,0.26990291262135924,0.388091294618729,63.40715775907203,1.533531229032464,-0.04280701383600338,-0.2506256293768128,0.13248029604308875,0.34575932485666866,1.0682632618314676,10.232210386899137,0.325215030244,0.8859140743739999,0.154849728193,Classical_Cepheid
+243412,0.6435000000000004,0.13967315443062386,0.3247627822515256,0.5190707271794328,0.6752585427537589,0.879866196783224,23.75478926785315,13.101,12.205,0.1120000000000001,11.814,0.5618730179775074,0.24202626641651032,0.5722326454033771,0.5782757551700131,37.33075381991498,3.2763034282434744,-0.049484189752863755,1.000270086380876,0.25145986677693544,1.1851795344480516,0.881970420711926,12.307386543586608,0.727226591713,2.6890731178099996,0.9426339697600001,W_Ursae_Maj
+247327,2.1945,0.2857724132471718,0.48556349263806586,0.6426319146340265,0.7897408686080372,0.9233730119805591,76.71641791148072,12.278,9.3305,1.089500000000001,7.889,2.7722455418473766,0.4024896265560166,0.17427385892116182,3.2994822291143238,348.582432044199,3.453919776991032,3.33954175839345,0.43017754594581625,1.2797726669476512,11.616459809345399,0.9751156767991696,9.49116371282888,0.8813121611779999,2.48443065817,0.7625254024889999,Mira
+257141,0.46950000000000003,0.13911916977845062,0.2554956670050432,0.39335583988843975,0.5357113476422322,0.7345991397031828,0.31574688939982387,13.869,13.295,0.08799999999999919,12.93,0.41061374975941844,0.2926315789473684,0.5305263157894737,0.414012881455879,27.448091498509122,1.9335536940956592,0.11275583542993171,0.5536755308508332,0.13923623640807684,0.18631078004976703,0.9589344760255665,13.303436442164505,0.8813121611779999,2.48443065817,0.7625254024889999,W_Ursae_Maj

cesium_app/tests/frontend/test_features.py

@@ -177,8 +177,31 @@ def test_delete_featureset(driver, project, dataset, featureset):
     driver.find_element_by_partial_link_text('Delete').click()
     driver.wait_for_xpath("//div[contains(text(),'Feature set deleted')]")
     try:
-        el = driver.wait_for_xpath("//td[contains(text(),'{test_featureset_name}')]")
+        el = driver.wait_for_xpath(f"//td[contains(text(),'{test_featureset_name}')]")
     except TimeoutException:
         pass
     else:
         raise Exception("Featureset still present in table after delete.")
+
+
+def test_upload_precomputed_features(driver, project, dataset):
+    driver.get('/')
+    driver.refresh()
+    proj_select = Select(driver.find_element_by_css_selector('[name=project]'))
+    proj_select.select_by_value(str(project.id))
+
+    driver.find_element_by_id('react-tabs-4').click()
+    driver.find_element_by_partial_link_text('Upload Pre-Computed Features')\
+          .click()
+    ds_select = Select(driver.find_element_by_css_selector('[name=datasetID]'))
+    ds_select.select_by_value(str(dataset.id))
+
+    fs_name = driver.find_element_by_css_selector('[name=featuresetName]')
+    fs_name.send_keys(test_featureset_name)
+
+    file_field = driver.find_element_by_css_selector('[name=dataFile]')
+    file_field.send_keys(pjoin(os.path.dirname(os.path.dirname(__file__)),
+                               'data', 'downloaded_cesium_featureset.csv'))
+    driver.find_element_by_class_name('btn-primary').click()
+
+    driver.wait_for_xpath(f"//td[contains(text(),'{test_featureset_name}')]")

cesium_app/tests/frontend/test_pipeline_sequentially.py

@@ -1,6 +1,7 @@
 import pytest
 from selenium import webdriver
 from selenium.webdriver.support.ui import Select
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
 import uuid
 import os
 from os.path import join as pjoin
@@ -10,6 +11,11 @@
 def test_pipeline_sequentially(driver):
     driver.get("/")
 
+    # Delete existing project if present
+    try:
+        driver.wait_for_xpath('//*[contains(text(), "Delete Project")]').click()
+    except (NoSuchElementException, TimeoutException):
+        pass
     # Add new project
     driver.wait_for_xpath('//*[contains(text(), "Or click here to add a new one")]').click()
 

cesium_app/tests/frontend/test_pipeline_sequentially_precomputed_features.py

@@ -0,0 +1,125 @@
+import pytest
+from selenium import webdriver
+from selenium.webdriver.support.ui import Select
+import uuid
+import os
+from os.path import join as pjoin
+import time
+
+
+def test_pipeline_sequentially_precomputed_features(driver):
+    driver.get("/")
+
+    # Add new project
+    driver.wait_for_xpath(
+        '//*[contains(text(), "Or click here to add a new one")]').click()
+
+    project_name = driver.find_element_by_css_selector('[name=projectName]')
+    test_proj_name = str(uuid.uuid4())
+    project_name.send_keys(test_proj_name)
+    project_desc = driver.find_element_by_css_selector(
+        '[name=projectDescription]')
+    project_desc.send_keys("Test Description")
+
+    driver.find_element_by_class_name('btn-primary').click()
+
+    status_td = driver.wait_for_xpath(
+        "//div[contains(text(),'Added new project')]")
+    driver.refresh()
+
+    # Ensure new project is selected
+    proj_select = Select(driver.find_element_by_css_selector('[name=project]'))
+    proj_select.select_by_visible_text(test_proj_name)
+
+    # Add new dataset
+    test_dataset_name = str(uuid.uuid4())
+    driver.find_element_by_id('react-tabs-2').click()
+    driver.find_element_by_partial_link_text('Upload new dataset').click()
+
+    dataset_name = driver.find_element_by_css_selector('[name=datasetName]')
+    dataset_name.send_keys(test_dataset_name)
+
+    header_file = driver.find_element_by_css_selector('[name=headerFile]')
+    header_file.send_keys(pjoin(
+        os.path.dirname(os.path.dirname(__file__)), 'data',
+        'larger_asas_training_subset_classes_with_metadata.dat'))
+
+    tar_file = driver.find_element_by_css_selector('[name=tarFile]')
+    tar_file.send_keys(pjoin(os.path.dirname(os.path.dirname(__file__)), 'data',
+                             'larger_asas_training_subset.tar.gz'))
+
+    driver.find_element_by_class_name('btn-primary').click()
+
+    status_td = driver.wait_for_xpath(
+        "//div[contains(text(),'Successfully uploaded new dataset')]")
+    driver.refresh()
+
+    # Ensure new project is selected
+    proj_select = Select(driver.find_element_by_css_selector('[name=project]'))
+    proj_select.select_by_visible_text(test_proj_name)
+
+    # Generate new feature set
+    test_featureset_name = str(uuid.uuid4())
+    driver.find_element_by_id('react-tabs-4').click()
+    driver.find_element_by_partial_link_text('Upload Pre-Computed Features')\
+          .click()
+
+    featureset_name = driver.find_element_by_css_selector(
+        '[name=featuresetName]')
+    featureset_name.send_keys(test_featureset_name)
+
+    # Ensure dataset from previous step is selected
+    dataset_select = Select(driver.find_element_by_css_selector(
+        '[name=datasetID]'))
+    dataset_select.select_by_visible_text(test_dataset_name)
+
+    file_field = driver.find_element_by_css_selector('[name=dataFile]')
+    file_field.send_keys(pjoin(os.path.dirname(os.path.dirname(__file__)),
+                               'data', 'downloaded_cesium_featureset.csv'))
+
+    driver.find_element_by_class_name('btn-primary').click()
+    status_td = driver.wait_for_xpath(
+        "//div[contains(text(),'Successfully uploaded new feature set')]")
+    status_td = driver.wait_for_xpath("//td[contains(text(),'Completed')]", 30)
+
+    # Build new model
+    driver.find_element_by_id('react-tabs-6').click()
+    driver.find_element_by_partial_link_text('Create New Model').click()
+
+    model_select = Select(driver.find_element_by_css_selector(
+        '[name=modelType]'))
+    model_select.select_by_visible_text('RandomForestClassifier (fast)')
+
+    model_name = driver.find_element_by_css_selector('[name=modelName]')
+    test_model_name = str(uuid.uuid4())
+    model_name.send_keys(test_model_name)
+
+    # Ensure featureset from previous step is selected
+    fset_select = Select(driver.find_element_by_css_selector(
+        '[name=featureset]'))
+    fset_select.select_by_visible_text(test_featureset_name)
+
+    driver.find_element_by_class_name('btn-primary').click()
+
+    driver.wait_for_xpath("//div[contains(text(),'Model training begun')]")
+
+    driver.wait_for_xpath("//td[contains(text(),'Completed')]", 60)
+
+    # Predict using dataset and model from this test
+    driver.find_element_by_id('react-tabs-8').click()
+    driver.find_element_by_partial_link_text('Predict Targets').click()
+
+    # Ensure model from previous step is selected
+    model_select = Select(driver.find_element_by_css_selector('[name=modelID]'))
+    model_select.select_by_visible_text(test_model_name)
+
+    # Ensure dataset from previous step is selected
+    dataset_select = Select(driver.find_element_by_css_selector(
+        '[name=datasetID]'))
+    dataset_select.select_by_visible_text(test_dataset_name)
+
+    driver.find_element_by_class_name('btn-primary').click()
+
+    driver.wait_for_xpath("//div[contains(text(),'Model predictions begun')]")
+
+    driver.wait_for_xpath("//td[contains(text(),'Completed')]", 20)

requirements.txt

@@ -1,4 +1,4 @@
-cesium>=0.9.4
+cesium>=0.9.5
 joblib>=0.11
 bokeh==0.12.5
 pytest-randomly

static/js/actions.js

@@ -438,6 +438,53 @@ export function computeFeatures(form) {
 }
 
 
+export function uploadFeatureset(form, currentProject) {
+  function fileReaderPromise(formFields, fileName, binary = false) {
+    return new Promise(resolve => {
+      const filereader = new FileReader();
+      if (binary) {
+        filereader.readAsDataURL(formFields[fileName][0]);
+      } else {
+        filereader.readAsText(formFields[fileName][0]);
+      }
+      filereader.onloadend = () => resolve({ body: filereader.result,
+                                             name: formFields[fileName][0].name });
+    });
+  }
+  form['projectID'] = currentProject.id;
+
+  return dispatch =>
+    promiseAction(
+      dispatch,
+      UPLOAD_DATASET,
+
+      fileReaderPromise(form, 'dataFile')
+        .then(data => {
+          form['dataFile'] = data;
+          return fetch('/precomputed_features', {
+            credentials: 'same-origin',
+            method: 'POST',
+            body: JSON.stringify(form),
+            headers: new Headers({
+              'Content-Type': 'application/json'
+            })
+          });
+        })
+        .then(response => response.json())
+        .then((json) => {
+          if (json.status == 'success') {
+            dispatch(showNotification('Successfully uploaded new feature set'));
+            dispatch(hideExpander('uploadFeatsFormExpander'));
+            dispatch(resetForm('uploadFeatures'));
+          } else {
+            return Promise.reject({ _error: json.message });
+          }
+          return json;
+        })
+    );
+}
+
+
 export function deleteDataset(id) {
   return dispatch =>
     promiseAction(