Merge pull request #181 from weecology/schema_field

Limit field names to ten characters

Author: henrykironde

App/Zooniverse/functions.R

@@ -166,7 +166,7 @@ plot_nests<-function(df, bird_df, MAPBOX_ACCESS_TOKEN){
 
   m<-leaflet(data=df) %>% 
     addProviderTiles("MapBox", layerId = "mapbox_id",options = providerTileOptions(id = mapbox_tileset, minZoom = 8, maxNativeZoom=24, maxZoom = 24, accessToken = MAPBOX_ACCESS_TOKEN)) %>%
-    addCircles(stroke = T,fillOpacity = 0.1,radius = 0.5,popup = ~htmlEscape(paste(round(sum_top1_s/num_obs_to,2),nest_id,sep=":"))) %>%
+    addCircles(stroke = T,fillOpacity = 0.1,radius = 0.5,popup = ~htmlEscape(paste(round(sum_top1/num_obs_to,2),nest_id,sep=":"))) %>%
     addCircles(data = bird_df, stroke = T, fillOpacity = 0, radius = 0.2, color = ~species_colors(label),
                popup = ~htmlEscape(paste(round(score,2), bird_id, sep=":")))
   return(m)
@@ -193,7 +193,7 @@ update_nests<-function(mapbox_tileset, df, bird_df, show_nests, show_birds,
   }
   if (show_nests) {
     map <- map %>%
-     addCircles(data=df,stroke = T,fillOpacity = 0.1,radius = 0.5,popup = ~htmlEscape(paste(round(sum_top1_s/num_obs_to,2),nest_id,sep=", ")))
+     addCircles(data=df,stroke = T,fillOpacity = 0.1,radius = 0.5,popup = ~htmlEscape(paste(round(sum_top1/num_obs_to,2),nest_id,sep=", ")))
   }
   if (show_birds) {
     map <- map %>%

Zooniverse/Snakefile

@@ -38,7 +38,7 @@ rule all:
                zip, site=SITES, year=YEARS, flight=FLIGHTS),
         expand(f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp",
                zip, site=SITES, year=YEARS),
-        expand(f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles",
+        expand(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles",
                zip, site=SITES, year=YEARS, flight=FLIGHTS)
 
 
@@ -131,7 +131,7 @@ rule upload_mapbox:
     input:
         f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles"
     output:
-        f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles"
+        touch(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles")
     conda:
         "EvergladesTools"
     shell:

Zooniverse/combine_nests.py

@@ -31,8 +31,8 @@ def load_shapefile(x):
         'last_obs': 'str',
         'num_obs': 'int',
         'species': 'str',
-        'sum_top1_s': 'float',
-        'num_obs_to': 'int',
+        'sum_top1': 'float',
+        'num_top1': 'int',
         'bird_match': 'str'
     })
     shp["site"] = get_site(x)

Zooniverse/nest_detection.py

@@ -68,12 +68,12 @@ def compare_site(gdf):
         # add target info to match
         row = geopandas.GeoDataFrame(pd.DataFrame(row).transpose(), crs=matches.crs)
         matches = geopandas.GeoDataFrame(pd.concat([matches, row], ignore_index=True))
-        matches["target_index"] = index
+        matches["target_ind"] = index
         matches = matches.rename(columns={
-            "xmin": "matched_xmin",
-            "max": "matched_xmax",
-            "ymin": "matched_ymin",
-            "ymax": "matched_ymax"
+            "xmin": "match_xmin",
+            "xmax": "match_xmax",
+            "ymin": "match_ymin",
+            "ymax": "match_ymax"
         })
 
         results.append(matches)
@@ -82,8 +82,8 @@ def compare_site(gdf):
         results = pd.concat(results)
     else:
         results = pd.DataFrame(columns=[
-            'matched_xmin', 'matched_ymin', 'xmax', 'matched_ymax', 'label', 'score', 'image_path', 'Date', 'bird_id',
-            'target_index', 'geometry'
+            'match_xmin', 'match_ymin', 'match_xmax', 'match_ymax', 'label', 'score', 'image_path', 'Site', 'Date',
+            'Year', 'event', 'file_posts', 'bird_id', 'target_ind'
         ])
 
     return results
@@ -102,19 +102,20 @@ def detect_nests(bird_detection_file, year, site, savedir):
     schema = {
         "geometry": "Polygon",
         "properties": {
-            'matched_xmin': 'float',
-            'matched_ymin': 'float',
-            'xmax': 'float',
-            'matched_ymax': 'float',
+            'match_xmin': 'float',
+            'match_ymin': 'float',
+            'match_xmax': 'float',
+            'match_ymax': 'float',
             'label': 'str',
             'score': 'float',
             'image_path': 'str',
             'Site': 'str',
             'Date': 'str',
             'Year': 'str',
-            'bird_id': 'int',
             'event': 'str',
-            'target_index': 'int'
+            'file_posts': 'str',
+            'bird_id': 'int',
+            'target_ind': 'int'
         }
     }
     if not results.empty:

Zooniverse/process_nests.py

@@ -55,13 +55,13 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
             top_score_data = summed_scores[summed_scores['sum'] == max(summed_scores['sum'])].reset_index()
             nest_info = nest_data.groupby(['Site', 'Year', 'target_ind']).agg({
                 'Date': ['min', 'max', 'count'],
-                'matched_xm': ['mean'],
-                'matched_ym': ['mean'],
-                'xmax': ['mean'],
-                'matched__1': ['mean']
+                'match_xmin': ['mean'],
+                'match_ymin': ['mean'],
+                'match_xmax': ['mean'],
+                'match_ymax': ['mean']
             }).reset_index()
-            xmean = (nest_info['matched_xm']['mean'][0] + nest_info['xmax']['mean']) / 2
-            ymean = (nest_info['matched_ym']['mean'][0] + nest_info['matched__1']['mean']) / 2
+            xmean = (nest_info['match_xmin']['mean'][0] + nest_info['match_xmax']['mean']) / 2
+            ymean = (nest_info['match_ymin']['mean'][0] + nest_info['match_ymax']['mean']) / 2
             bird_match = ",".join([str(x) for x in nest_data["bird_id"]])
             nests.append([
                 target_ind, nest_info['Site'][0], nest_info['Year'][0], xmean[0], ymean[0], nest_info['Date']['min'][0],
@@ -77,7 +77,7 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
         nests = pd.DataFrame(nests,
                              columns=[
                                  'nest_id', 'Site', 'Year', 'xmean', 'ymean', 'first_obs', 'last_obs', 'num_obs',
-                                 'species', 'sum_top1_score', 'num_obs_top1', 'bird_match'
+                                 'species', 'sum_top1', 'num_top1', 'bird_match'
                              ])
         nests_shp = geopandas.GeoDataFrame(nests, geometry=geopandas.points_from_xy(nests.xmean, nests.ymean))
         nests_shp.crs = nests_data.crs
@@ -95,8 +95,8 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
                 'last_obs': 'str',
                 'num_obs': 'int',
                 'species': 'str',
-                'sum_top1_score': 'float',
-                'num_obs_top1': 'int',
+                'sum_top1': 'float',
+                'num_top1': 'int',
                 'bird_match': 'str'
             }
         }

Zooniverse/upload_mapbox.py

@@ -56,7 +56,7 @@ def retrieve_upload_status(self, upload_id):
 
 def get_credentials():
     """Get credentials from mapbox.ini"""
-    with open("/blue/ewhite/everglades/mapbox/mapbox.ini", "rb") as f:
+    with open("/blue/ewhite/everglades/mapbox.ini", "rb") as f:
         toml_dict = tomli.load(f)
         access_token = toml_dict['mapbox']['access-token']
     return access_token

everglades_workflow.sh

@@ -4,7 +4,7 @@
 #SBATCH --mail-type=FAIL
 #SBATCH --gpus=a100:4
 #SBATCH --cpus-per-task=60
-#SBATCH --mem=1200gb
+#SBATCH --mem=600gb
 #SBATCH --time=80:00:00
 #SBATCH --partition=gpu
 #SBATCH --output=/blue/ewhite/everglades/EvergladesTools/logs/everglades_workflow.out