Merge pull request #201 from haiwen/support_byte_range_upload

Add byte-range upload.

Author: killing

common/rpc-service.c

@@ -5293,6 +5293,33 @@ seafile_org_get_shared_users_by_repo (int org_id,
                                                             org_id, repo_id);
 }
 
+/* Resumable file upload. */
+
+gint64
+seafile_get_upload_tmp_file_offset (const char *repo_id, const char *file_path,
+                                    GError **error)
+{
+    if (!repo_id || !is_uuid_valid(repo_id)) {
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS,
+                     "Invalid repo id");
+        return -1;
+    }
+
+    int path_len;
+    if (!file_path || (path_len = strlen(file_path)) == 0) {
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS,
+                     "Invalid file path");
+        return -1;
+    }
+
+    char *rfile_path = format_dir_path (file_path);
+    gint64 ret = seaf_repo_manager_get_upload_tmp_file_offset (seaf->repo_mgr, repo_id,
+                                                               rfile_path, error);
+    g_free (rfile_path);
+
+    return ret;
+}
+
 char *
 seafile_convert_repo_path (const char *repo_id,
                            const char *path,

include/seafile-rpc.h

@@ -1103,6 +1103,10 @@ seafile_org_get_shared_users_by_repo (int org_id,
                                       const char *repo_id,
                                       GError **error);
 
+gint64
+seafile_get_upload_tmp_file_offset (const char *repo_id, const char *file_path,
+                                    GError **error);
+
 char *
 seafile_convert_repo_path (const char *repo_id,
                            const char *path,

python/seafile/rpcclient.py

@@ -989,6 +989,11 @@ def get_file_count_info_by_path(repo_id, path):
     def get_trash_repo_owner(repo_id):
         pass
 
+    @searpc_func("int64", ["string", "string"])
+    def seafile_get_upload_tmp_file_offset(repo_id, file_path):
+        pass
+    get_upload_tmp_file_offset = seafile_get_upload_tmp_file_offset
+
     @searpc_func("int", ["string", "string", "string", "string"])
     def seafile_mkdir_with_parents (repo_id, parent_dir, relative_path, username):
         pass

python/seaserv/api.py

@@ -383,6 +383,9 @@ def set_repo_history_limit(self, repo_id, days):
     def check_repo_blocks_missing(self, repo_id, blklist):
         return seafserv_threaded_rpc.check_repo_blocks_missing(repo_id, blklist)
 
+    def get_upload_tmp_file_offset (self, repo_id, file_path):
+        return seafserv_threaded_rpc.get_upload_tmp_file_offset (repo_id, file_path)
+
     # file lock
     def check_file_lock(self, repo_id, path, user):
         """

server/http-server.c

@@ -35,6 +35,7 @@
 #define DEFAULT_MAX_INDEXING_THREADS 1
 #define DEFAULT_MAX_INDEX_PROCESSING_THREADS 3
 #define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */
+#define DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE 0600
 
 #define HOST "host"
 #define PORT "port"
@@ -130,6 +131,7 @@ load_http_config (HttpServerStruct *htp_server, SeafileSession *session)
     char *encoding;
     int max_indexing_threads;
     int max_index_processing_threads;
+    char *cluster_shared_temp_file_mode = NULL;
 
     host = fileserver_config_get_string (session->config, HOST, &error);
     if (!error) {
@@ -232,6 +234,28 @@ load_http_config (HttpServerStruct *htp_server, SeafileSession *session)
     seaf_message ("fileserver: max_index_processing_threads= %d\n",
                   htp_server->max_index_processing_threads);
 
+    cluster_shared_temp_file_mode = fileserver_config_get_string (session->config,
+                                                                  "cluster_shared_temp_file_mode",
+                                                                  &error);
+    if (error) {
+        htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE;
+        g_clear_error (&error);
+    } else {
+        if (!cluster_shared_temp_file_mode) {
+            htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE;
+        } else {
+            htp_server->cluster_shared_temp_file_mode = strtol(cluster_shared_temp_file_mode, NULL, 8);
+
+            if (htp_server->cluster_shared_temp_file_mode < 0001 ||
+                htp_server->cluster_shared_temp_file_mode > 0777)
+                htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE;
+
+            g_free (cluster_shared_temp_file_mode);
+        }
+    }
+    seaf_message ("fileserver: cluster_shared_temp_file_mode = %o\n",
+                  htp_server->cluster_shared_temp_file_mode);
+
     encoding = g_key_file_get_string (session->config,
                                       "zip", "windows_encoding",
                                       &error);

server/http-server.h

@@ -21,6 +21,7 @@ struct _HttpServerStruct {
     int max_indexing_threads;
     int worker_threads;
     int max_index_processing_threads;
+    int cluster_shared_temp_file_mode;
 };
 
 typedef struct _HttpServerStruct HttpServerStruct;

server/repo-mgr.c

@@ -1058,6 +1058,12 @@ create_tables_mysql (SeafRepoManager *mgr)
     if (seaf_db_query (db, sql) < 0)
         return -1;
 
+    sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles ( "
+        "id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, repo_id CHAR(40) NOT NULL, "
+        "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL) ENGINE=INNODB";
+    if (seaf_db_query (db, sql) < 0)
+        return -1;
+
     return 0;
 }
 
@@ -1206,6 +1212,11 @@ create_tables_sqlite (SeafRepoManager *mgr)
     if (seaf_db_query (db, sql) < 0)
         return -1;
 
+    sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles (repo_id CHAR(40) NOT NULL, "
+        "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL)";
+    if (seaf_db_query (db, sql) < 0)
+        return -1;
+
     return 0;
 }
 
@@ -1344,6 +1355,11 @@ create_tables_pgsql (SeafRepoManager *mgr)
     if (seaf_db_query (db, sql) < 0)
         return -1;
 
+    sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles (repo_id CHAR(40) NOT NULL, "
+        "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL)";
+    if (seaf_db_query (db, sql) < 0)
+        return -1;
+
     sql = "CREATE TABLE IF NOT EXISTS RepoInfo (repo_id CHAR(36) PRIMARY KEY, "
         "name VARCHAR(255) NOT NULL, update_time BIGINT, version INTEGER, "
         "is_encrypted INTEGER, last_modifier VARCHAR(255))";
@@ -3918,6 +3934,119 @@ seaf_get_total_storage (GError **error)
     return size;
 }
 
+int
+seaf_repo_manager_add_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       const char *tmp_file,
+                                       GError **error)
+{
+    int ret = seaf_db_statement_query (mgr->seaf->db,
+                                       "INSERT INTO WebUploadTempFiles "
+                                       "(repo_id, file_path, tmp_file_path) "
+                                       "VALUES (?, ?, ?)", 3, "string", repo_id,
+                                       "string", file_path, "string", tmp_file);
+
+    if (ret < 0) {
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
+                     "Failed to add upload tmp file record to db.");
+    }
+
+    return ret;
+}
+
+int
+seaf_repo_manager_del_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       GError **error)
+{
+    int ret = seaf_db_statement_query (mgr->seaf->db,
+                                       "DELETE FROM WebUploadTempFiles WHERE "
+                                       "repo_id = ? AND file_path = ?",
+                                       2, "string", repo_id, "string", file_path);
+    if (ret < 0) {
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
+                     "Failed to delete upload tmp file record from db.");
+    }
+
+    return ret;
+}
+
+static gboolean
+get_tmp_file_path (SeafDBRow *row, void *data)
+{
+    char **path = data;
+
+    *path = g_strdup (seaf_db_row_get_column_text (row, 0));
+
+    return FALSE;
+}
+
+char *
+seaf_repo_manager_get_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       GError **error)
+{
+    char *tmp_file_path = NULL;
+
+    int ret = seaf_db_statement_foreach_row (mgr->seaf->db,
+                                             "SELECT tmp_file_path FROM WebUploadTempFiles "
+                                             "WHERE repo_id = ? AND file_path = ?",
+                                             get_tmp_file_path, &tmp_file_path,
+                                             2, "string", repo_id, "string", file_path);
+    if (ret < 0) {
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
+                     "Failed to get upload temp file path from db.");
+        return NULL;
+    }
+
+    return tmp_file_path;
+}
+
+gint64
+seaf_repo_manager_get_upload_tmp_file_offset (SeafRepoManager *mgr,
+                                              const char *repo_id,
+                                              const char *file_path,
+                                              GError **error)
+{
+    char *tmp_file_path = NULL;
+    SeafStat file_stat;
+
+    tmp_file_path = seaf_repo_manager_get_upload_tmp_file (mgr, repo_id,
+                                                           file_path, error);
+    if (*error) {
+        return -1;
+    }
+
+    if (!tmp_file_path)
+        return 0;
+
+    if (seaf_stat (tmp_file_path, &file_stat) < 0) {
+        if (errno == ENOENT) {
+            seaf_message ("Temp file %s doesn't exist, remove reocrd from db.\n",
+                          tmp_file_path);
+            if (seaf_repo_manager_del_upload_tmp_file (mgr, repo_id,
+                                                       file_path, error) < 0) {
+                g_free (tmp_file_path);
+                return -1;
+            }
+            return 0;
+        }
+        seaf_warning ("Failed to stat temp file %s: %s.\n",
+                      tmp_file_path, strerror(errno));
+        g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL,
+                     "Failed to stat temp file.");
+        g_free (tmp_file_path);
+        return -1;
+    }
+
+    g_free (tmp_file_path);
+
+    return file_stat.st_size;
+}
+
 void
 seaf_repo_manager_update_repo_info (SeafRepoManager *mgr,
                                     const char *repo_id, const char *head_commit_id)

server/repo-mgr.h

@@ -818,6 +818,31 @@ seaf_get_total_file_number (GError **error);
 gint64
 seaf_get_total_storage (GError **error);
 
+int
+seaf_repo_manager_add_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       const char *tmp_file,
+                                       GError **error);
+
+int
+seaf_repo_manager_del_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       GError **error);
+
+char *
+seaf_repo_manager_get_upload_tmp_file (SeafRepoManager *mgr,
+                                       const char *repo_id,
+                                       const char *file_path,
+                                       GError **error);
+
+gint64
+seaf_repo_manager_get_upload_tmp_file_offset (SeafRepoManager *mgr,
+                                              const char *repo_id,
+                                              const char *file_path,
+                                              GError **error);
+
 void
 seaf_repo_manager_update_repo_info (SeafRepoManager *mgr,
                                     const char *repo_id,

server/seaf-server.c

@@ -587,6 +587,11 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode)
                                      "get_virtual_repo",
                                      searpc_signature_object__string_string_string());
 
+    searpc_server_register_function ("seafserv-threaded-rpcserver",
+                                     seafile_get_upload_tmp_file_offset,
+                                     "seafile_get_upload_tmp_file_offset",
+                                     searpc_signature_int64__string_string());
+
     /* Clean trash */
 
     searpc_server_register_function ("seafserv-threaded-rpcserver",